{"id":5900,"date":"2017-08-08T15:10:04","date_gmt":"2017-08-08T12:10:04","guid":{"rendered":"http:\/\/ssrlab.by\/?p=5900"},"modified":"2020-12-15T12:50:49","modified_gmt":"2020-12-15T09:50:49","slug":"takenizatar","status":"publish","type":"post","link":"https:\/\/ssrlab.by\/ru\/5900","title":{"rendered":"(\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f) \u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440"},"content":{"rendered":"<p class=\"qtranxs-available-languages-message qtranxs-available-languages-message-ru\">\u0418\u0437\u0432\u0438\u043d\u0438\u0442\u0435, \u044d\u0442\u043e\u0442 \u0442\u0435\u0445\u0442 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u0442\u043e\u043b\u044c\u043a\u043e \u0432 &ldquo;<a href=\"https:\/\/ssrlab.by\/BY\/wp-json\/wp\/v2\/posts\/5900\" class=\"qtranxs-available-language-link qtranxs-available-language-link-BY\" title=\"\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f\">\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f<\/a>&rdquo; \u0438 &ldquo;<a href=\"https:\/\/ssrlab.by\/en\/wp-json\/wp\/v2\/posts\/5900\" class=\"qtranxs-available-language-link qtranxs-available-language-link-en\" title=\"English\">\u0410\u043c\u0435\u0440\u0438\u043a\u0430\u043d\u0441\u043a\u0438\u0439 \u0410\u043d\u0433\u043b\u0438\u0439\u0441\u043a\u0438\u0439<\/a>&rdquo;. For the sake of viewer convenience, the content is shown below in this site default language. You may click one of the links to switch the site language to another available language.<\/p><p>\u0421\u044d\u0440\u0432\u0456\u0441\u00a0<strong>\u00ab<a href=\"https:\/\/corpus.by\/Tokenizer\/?lang=be\" target=\"_blank\" rel=\"noopener noreferrer\">\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440<\/a>\u00bb<\/strong> \u043f\u0440\u044b\u0437\u043d\u0430\u0447\u0430\u043d\u044b \u0434\u043b\u044f \u0432\u044b\u043b\u0443\u0447\u044d\u043d\u043d\u044f \u045e \u0442\u044d\u043a\u0441\u0446\u0435 \u0442\u043e\u043a\u0435\u043d\u0430\u045e. \u041d\u0430 \u045e\u0432\u0430\u0445\u043e\u0434 \u0441\u044d\u0440\u0432\u0456\u0441\u0443 \u043f\u0430\u0434\u0430\u0435\u0446\u0446\u0430 \u0442\u044d\u043a\u0441\u0442, \u044f\u043a\u0456 \u043f\u0430\u0442\u0440\u0430\u0431\u0443\u0435 \u0442\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0446\u044b\u0456. \u041f\u0430\u0441\u043b\u044f \u0430\u043f\u0440\u0430\u0446\u043e\u045e\u043a\u0456 \u0442\u044d\u043a\u0441\u0442\u0443 \u043d\u0430 \u0432\u044b\u0445\u0430\u0434\u0437\u0435 \u043a\u0430\u0440\u044b\u0441\u0442\u0430\u043b\u044c\u043d\u0456\u043a \u0430\u0442\u0440\u044b\u043c\u043b\u0456\u0432\u0430\u0435 \u0441\u043f\u0456\u0441 \u0432\u044b\u043b\u0443\u0447\u0430\u043d\u044b\u0445 \u0442\u043e\u043a\u0435\u043d\u0430\u045e.<\/p>\n<p>&nbsp;<\/p>\n<p><strong>\u0410\u0441\u043d\u043e\u045e\u043d\u044b\u044f \u0442\u044d\u0440\u043c\u0456\u043d\u044b \u0456 \u043f\u0430\u043d\u044f\u0446\u0446\u0456<\/strong><\/p>\n<p><i>\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0446\u044b\u044f <\/i>(\u0430\u043d\u0433\u043b.<i> tokenizing<\/i>, \u043b\u0435\u043a\u0441\u0456\u0447\u043d\u044b \u0430\u043d\u0430\u043b\u0456\u0437)<i> \u2014\u00a0<\/i>\u0432\u044b\u043b\u0443\u0447\u044d\u043d\u043d\u0435 \u045e \u0442\u044d\u043a\u0441\u0446\u0435 \u0442\u043e\u043a\u0435\u043d\u0430\u045e \u0441\u043f\u0435\u0446\u044b\u044f\u043b\u044c\u043d\u0430\u0439 \u043a\u0430\u043c\u043f\u2019\u044e\u0442\u0430\u0440\u043d\u0430\u0439 \u043f\u0440\u0430\u0433\u0440\u0430\u043c\u0430\u0439. \u0413\u044d\u0442\u0430 \u043f\u0440\u0430\u0446\u044d\u0441 \u0430\u043d\u0430\u043b\u0456\u0442\u044b\u0447\u043d\u0430\u0433\u0430 \u0440\u0430\u0437\u0431\u043e\u0440\u0443 \u045e\u0432\u0430\u0445\u043e\u0434\u043d\u0430\u0439 \u043f\u0430\u0441\u043b\u044f\u0434\u043e\u045e\u043d\u0430\u0441\u0446\u0456 \u0437\u043d\u0430\u043a\u0430\u045e \u043d\u0430 \u0440\u0430\u0441\u043f\u0430\u0437\u043d\u0430\u043d\u044b\u044f \u0433\u0440\u0443\u043f\u044b \u2014 \u043b\u0435\u043a\u0441\u0435\u043c\u044b, \u0437 \u043c\u044d\u0442\u0430\u0439 \u0430\u0442\u0440\u044b\u043c\u0430\u043d\u043d\u044f \u043d\u0430 \u0432\u044b\u0445\u0430\u0434\u0437\u0435 \u0456\u0434\u044d\u043d\u0442\u044b\u0444\u0456\u043a\u0430\u0432\u0430\u043d\u044b\u0445 \u043f\u0430\u0441\u043b\u044f\u0434\u043e\u045e\u043d\u0430\u0441\u0446\u044f\u045e, \u0442\u0430\u043a \u0437\u0432\u0430\u043d\u044b\u0445 \u00ab\u0442\u043e\u043a\u0435\u043d\u0430\u045e\u00bb.\u00a0\u041b\u0435\u043a\u0441\u0456\u0447\u043d\u044b \u0430\u043d\u0430\u043b\u0456\u0437 \u0432\u044b\u043a\u0430\u0440\u044b\u0441\u0442\u043e\u045e\u0432\u0430\u0435\u0446\u0446\u0430 \u045e \u043a\u0430\u043c\u043f\u0456\u043b\u044f\u0442\u0430\u0440\u0430\u0445 \u0456 \u0456\u043d\u0442\u044d\u0440\u043f\u0440\u044d\u0442\u0430\u0442\u0430\u0440\u0430\u0445 \u0437\u044b\u0445\u043e\u0434\u043d\u0430\u0433\u0430 \u043a\u043e\u0434\u0443 \u043c\u043e\u045e \u043f\u0440\u0430\u0433\u0440\u0430\u043c\u0430\u0432\u0430\u043d\u043d\u044f, \u0456 \u045e \u0440\u043e\u0437\u043d\u044b\u0445 \u043f\u0430\u0440\u0441\u0435\u0440\u0430\u0445 \u0441\u043b\u043e\u045e \u043d\u0430\u0442\u0443\u0440\u0430\u043b\u044c\u043d\u044b\u0445 \u043c\u043e\u045e.<\/p>\n<p><em>\u0422\u043e\u043a\u0435\u043d<\/em> (\u043b\u0435\u043a\u0441\u0456\u0447\u043d\u044b \u0430\u043d\u0430\u043b\u0456\u0437) \u2014\u00a0\u043f\u0430\u0441\u043b\u044f\u0434\u043e\u045e\u043d\u0430\u0441\u0446\u044c \u0437\u043d\u0430\u043a\u0430\u045e \u0443 \u043b\u0435\u043a\u0441\u0456\u0447\u043d\u044b\u043c \u0430\u043d\u0430\u043b\u0456\u0437\u0435 \u045e \u0456\u043d\u0444\u0430\u0440\u043c\u0430\u0442\u044b\u0446\u044b, \u0430\u0434\u043f\u0430\u0432\u0435\u0434\u043d\u044b \u043b\u0435\u043a\u0441\u0435\u043c\u0435. \u0410\u0431\u2019\u0435\u043a\u0442, \u044f\u043a\u0456 \u0441\u0442\u0432\u0430\u0440\u0430\u0435\u0446\u0446\u0430 \u0437 \u043b\u0435\u043a\u0441\u0435\u043c\u044b \u045e \u043f\u0440\u0430\u0446\u044d\u0441\u0435 \u043b\u0435\u043a\u0441\u0456\u0447\u043d\u0430\u0433\u0430 \u0430\u043d\u0430\u043b\u0456\u0437\u0443 (\u0442\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0446\u044b\u0456).<\/p>\n<p><em>\u0428\u0430\u0431\u043b\u043e\u043d \u0442\u043e\u043a\u0435\u043d\u0430<\/em> \u2014 \u0444\u0430\u0440\u043c\u0430\u043b\u044c\u043d\u0430\u0435 \u0430\u043f\u0456\u0441\u0430\u043d\u043d\u0435 \u043a\u043b\u0430\u0441\u0443 \u043b\u0435\u043a\u0441\u0435\u043c, \u044f\u043a\u0456\u044f \u043c\u043e\u0433\u0443\u0446\u044c \u0441\u0442\u0432\u0430\u0440\u044b\u0446\u044c \u0434\u0430\u0434\u0437\u0435\u043d\u044b \u0442\u044b\u043f \u0442\u043e\u043a\u0435\u043d\u0430.<\/p>\n<p>&nbsp;<\/p>\n<p><strong>\u0410\u043f\u0456\u0441\u0430\u043d\u043d\u0435 \u043a\u0430\u0440\u044b\u0441\u0442\u0430\u043b\u044c\u043d\u0456\u0446\u043a\u0430\u0433\u0430 \u0456\u043d\u0442\u044d\u0440\u0444\u0435\u0439\u0441\u0443<\/strong><\/p>\n<p>\u041a\u0430\u0440\u044b\u0441\u0442\u0430\u043b\u044c\u043d\u0456\u0446\u043a\u0456 \u0456\u043d\u0442\u044d\u0440\u0444\u0435\u0439\u0441 \u0441\u044d\u0440\u0432\u0456\u0441\u0443 \u043f\u0440\u0430\u0434\u0441\u0442\u0430\u045e\u043b\u0435\u043d\u044b \u043d\u0430 \u043c\u0430\u043b\u044e\u043d\u043a\u0443 1.<\/p>\n<div id=\"attachment_5901\" style=\"width: 710px\" class=\"wp-caption alignnone\"><a href=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen01.png\" class=\"grouped_elements\" rel=\"tc-fancybox-group5900\" title=\"\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\"><img aria-describedby=\"caption-attachment-5901\" loading=\"lazy\" class=\"wp-image-5901\" src=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen01.png\" alt=\"\u041c\u0430\u043b\u044e\u043d\u0430\u043a 1. \u0413\u0440\u0430\u0444\u0456\u0447\u043d\u044b \u0456\u043d\u0442\u044d\u0440\u0444\u0435\u0439\u0441 \u0441\u044d\u0440\u0432\u0456\u0441\u0443 \u00ab\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\u00bb\" width=\"700\" height=\"259\" srcset=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen01.png 992w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen01-300x111.png 300w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen01-768x284.png 768w\" sizes=\"(max-width: 700px) 100vw, 700px\" \/><\/a><p id=\"caption-attachment-5901\" class=\"wp-caption-text\">\u041c\u0430\u043b\u044e\u043d\u0430\u043a 1. \u0413\u0440\u0430\u0444\u0456\u0447\u043d\u044b \u0456\u043d\u0442\u044d\u0440\u0444\u0435\u0439\u0441 \u0441\u044d\u0440\u0432\u0456\u0441\u0443 \u00ab\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\u00bb<\/p><\/div>\n<ul>\n<li>\u043f\u043e\u043b\u0435 \u045e\u0432\u043e\u0434\u0443 \u0442\u044d\u043a\u0441\u0442\u0443, \u044f\u043a\u0456 \u043f\u0430\u0442\u0440\u0430\u0431\u0443\u0435 \u0442\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0446\u044b\u0456;<\/li>\n<li>\u043c\u0435\u043d\u044e \u0432\u044b\u0431\u0430\u0440\u0443 \u043c\u044d\u0442\u0430\u0432\u0430\u0439 \u043c\u043e\u0432\u044b;<\/li>\n<li>\u043a\u043d\u043e\u043f\u043a\u0430 \u00ab\u0410\u0442\u0440\u044b\u043c\u0430\u0446\u044c \u0441\u043f\u0456\u0441 \u0442\u043e\u043a\u0435\u043d\u0430\u045e!\u00bb, \u044f\u043a\u0430\u044f \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0435 \u0442\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0446\u044b\u044e;<\/li>\n<li>\u043f\u043e\u043b\u0435 \u0432\u044b\u0432\u0430\u0434\u0443, \u0434\u0437\u0435 \u0430\u0434\u043b\u044e\u0441\u0442\u0440\u043e\u045e\u0432\u0430\u044e\u0446\u0446\u0430 \u0432\u044b\u043b\u0443\u0447\u0430\u043d\u044b\u044f \u0442\u043e\u043a\u0435\u043d\u044b.<\/li>\n<\/ul>\n<p><strong>\u041a\u0430\u0440\u044b\u0441\u0442\u0430\u043b\u044c\u043d\u0456\u0446\u043a\u0456 \u0441\u0446\u044d\u043d\u0430\u0440 \u043f\u0440\u0430\u0446\u044b \u0437 \u0441\u044d\u0440\u0432\u0456\u0441\u0430\u043c<\/strong><\/p>\n<ol>\n<li>\u0423\u0432\u0435\u0441\u0446\u0456 \u045e \u043f\u043e\u043b\u0435 \u045e\u0432\u043e\u0434\u0443 \u0442\u044d\u043a\u0441\u0442, \u044f\u043a\u0456 \u043f\u0430\u0442\u0440\u0430\u0431\u0443\u0435 \u0442\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0446\u044b\u0456.<\/li>\n<li>\u0412\u044b\u0431\u0440\u0430\u0446\u044c \u043c\u044d\u0442\u0430\u0432\u0443\u044e \u043c\u043e\u0432\u0443.<\/li>\n<li>\u041d\u0430\u0446\u0456\u0441\u043d\u0443\u0446\u044c \u043a\u043d\u043e\u043f\u043a\u0443 \u00ab\u0410\u0442\u0440\u044b\u043c\u0430\u0446\u044c \u0441\u043f\u0456\u0441 \u0442\u043e\u043a\u0435\u043d\u0430\u045e!\u00bb.<\/li>\n<li>\u041f\u0440\u0430\u0433\u043b\u044f\u0434\u0437\u0435\u0446\u044c \u0432\u044b\u043b\u0443\u0447\u0430\u043d\u044b\u044f \u0442\u043e\u043a\u0435\u043d\u044b (\u043c\u0430\u043b\u044e\u043d\u0430\u043a 2).<\/li>\n<\/ol>\n<div id=\"attachment_5902\" style=\"width: 710px\" class=\"wp-caption alignnone\"><a href=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02.png\" class=\"grouped_elements\" rel=\"tc-fancybox-group5900\" title=\"\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\"><img aria-describedby=\"caption-attachment-5902\" loading=\"lazy\" class=\"wp-image-5902\" src=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02.png\" alt=\"\u041c\u0430\u043b\u044e\u043d\u0430\u043a 2. \u0412\u044b\u043d\u0456\u043a\u0456 \u043f\u0440\u0430\u0446\u044b \u0441\u044d\u0440\u0432\u0456\u0441\u0430 \u00ab\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\u00bb: \u0432\u044b\u043b\u0443\u0447\u0430\u043d\u044b\u044f \u0442\u043e\u043a\u0435\u043d\u044b\" width=\"700\" height=\"394\" srcset=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02.png 801w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02-300x169.png 300w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02-768x432.png 768w\" sizes=\"(max-width: 700px) 100vw, 700px\" \/><\/a><p id=\"caption-attachment-5902\" class=\"wp-caption-text\">\u041c\u0430\u043b\u044e\u043d\u0430\u043a 2. \u0412\u044b\u043d\u0456\u043a\u0456 \u043f\u0440\u0430\u0446\u044b \u0441\u044d\u0440\u0432\u0456\u0441\u0430 \u00ab\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\u00bb: \u0432\u044b\u043b\u0443\u0447\u0430\u043d\u044b\u044f \u0442\u043e\u043a\u0435\u043d\u044b<\/p><\/div>\n<p>&nbsp;<\/p>\n<p><strong>\u0414\u043e\u0441\u0442\u0443\u043f \u0434\u0430 \u0441\u044d\u0440\u0432\u0456\u0441\u0430 \u043f\u0440\u0430\u0437 API<\/strong><\/p>\n<p>\u0414\u043b\u044f \u0434\u043e\u0441\u0442\u0443\u043f\u0443 \u0434\u0430 \u0441\u044d\u0440\u0432\u0456\u0441\u0430 \u201c\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\u201d \u043f\u0440\u0430\u0437 API, \u043d\u0435\u0430\u0431\u0445\u043e\u0434\u043d\u0430 \u0430\u0434\u043f\u0440\u0430\u0432\u0456\u0446\u044c AJAX-\u0437\u0430\u043f\u044b\u0442 \u0442\u044b\u043f\u0443 POST \u043d\u0430 \u0430\u0434\u0440\u0430\u0441 https:\/\/corpus.by\/Tokenizer\/api.php. \u041f\u0440\u0430\u0437 \u043c\u0430\u0441\u0456\u045e <strong><em>data<\/em><\/strong> \u043f\u0435\u0440\u0430\u0434\u0430\u044e\u0446\u0446\u0430 \u043d\u0430\u0441\u0442\u0443\u043f\u043d\u044b\u044f \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b:<\/p>\n<ul>\n<li><strong><em>inputText<\/em><\/strong>\u00a0\u2014 \u0430\u0434\u0432\u043e\u043b\u044c\u043d\u044b \u045e\u0432\u0430\u0445\u043e\u0434\u043d\u044b \u0442\u044d\u043a\u0441\u0442.<\/li>\n<li><strong><em>language<\/em><\/strong>\u00a0\u2014 \u043c\u043e\u0432\u0430 \u0442\u044d\u043a\u0441\u0442\u0443. \u0414\u0430\u0441\u0442\u0443\u043f\u043d\u044b \u043d\u0430\u0441\u0442\u0443\u043f\u043d\u044b\u044f \u043c\u043e\u0432\u044b:\n<ul>\n<li><em><strong>be<\/strong> <\/em>\u2014 \u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f \u043c\u043e\u0432\u0430;<\/li>\n<li><em><strong>en<\/strong> <\/em>\u2014 \u0430\u043d\u0433\u043b\u0456\u0439\u0441\u043a\u0430\u044f \u043c\u043e\u0432\u0430;<\/li>\n<li><em><strong>ru<\/strong> <\/em>\u2014 \u0440\u0443\u0441\u043a\u0430\u044f \u043c\u043e\u0432\u0430.<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u041f\u0440\u044b\u043a\u043b\u0430\u0434 AJAX-\u0437\u0430\u043f\u044b\u0442\u0443:<\/p>\n<blockquote><p><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">$.ajax({<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0type: &#8220;POST&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0url: &#8220;https:\/\/corpus.by\/Tokenizer\/api.php&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0data:{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;inputText&#8221;: &#8220;\u0421\u0456\u0301\u0440\u044b\u0443\u0441 (\u043b\u0430\u0446. Sirius), \u0442\u0430\u043a\u0441\u0430\u043c\u0430 \u0410\u043b\u044c\u0444\u0430 \u0412\u044f\u043b\u0456\u043a\u0430\u0433\u0430 \u0421\u0430\u0431\u0430\u043a\u0456 (\u043b\u0430\u0446. Alpha Canis Majoris) \u2013 \u043d\u0430\u0439\u044f\u0440\u0447\u044d\u0439\u0448\u0430\u044f \u0437\u043e\u0440\u043a\u0430 \u043d\u0430\u0447\u043d\u043e\u0433\u043e \u043d\u0435\u0431\u0430.&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;language&#8221;: &#8220;be&#8221;<\/span><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\"><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0success: function(msg){ },<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0error: function() { }<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">});<\/span><\/span><\/p><\/blockquote>\n<p>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0435\u0440\u043d\u0435 JSON-\u043c\u0430\u0441\u0456\u045e \u0437 \u0443\u0432\u0430\u0445\u043e\u0434\u043d\u044b\u043c \u0442\u044d\u043a\u0441\u0442\u0430\u043c (\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 <em><strong>text<\/strong><\/em>), \u043c\u0430\u0441\u0456\u0432\u0430\u043c \u0441\u0430 \u0437\u0433\u0440\u0443\u043f\u0430\u0432\u0430\u043d\u044b\u043c\u0456 \u0441\u0456\u043c\u0432\u0430\u043b\u044c\u043d\u044b\u043c\u0456 \u043f\u0430\u0441\u043b\u044f\u0434\u043e\u045e\u043d\u0430\u0441\u0446\u044f\u043c\u0456 (\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 <em><strong>CharacterArr<\/strong><\/em>) \u0456 \u043c\u0430\u0441\u0456\u0432\u0430\u043c \u0437 \u0441\u0456\u043c\u0432\u0430\u043b\u044c\u043d\u044b\u043c\u0456 \u043f\u0430\u0441\u043b\u044f\u0434\u043e\u045e\u043d\u0430\u0441\u0446\u044f\u043c\u0456, \u0437\u0433\u0440\u0443\u043f\u0430\u0432\u0430\u043d\u044b\u043c\u0456 \u043f\u0430\u0432\u043e\u0434\u043b\u0435 \u0434\u0430\u0434\u0430\u0442\u043a\u043e\u0432\u044b\u0445 \u043f\u0440\u0430\u0432\u0456\u043b, (\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 <em><strong>ResultArr<\/strong><\/em>). \u041d\u0430\u043f\u0440\u044b\u043a\u043b\u0430\u0434, \u043f\u0430 \u0432\u044b\u0448\u044d\u0439 \u043f\u0440\u044b\u0432\u0435\u0434\u0437\u0435\u043d\u044b\u043c AJAX-\u0437\u0430\u043f\u044b\u0446\u0435 \u0431\u0443\u0434\u0437\u0435 \u0441\u0444\u0430\u0440\u043c\u0456\u0440\u0430\u0432\u0430\u043d\u044b \u043d\u0430\u0441\u0442\u0443\u043f\u043d\u044b \u0430\u0434\u043a\u0430\u0437:<\/p>\n<blockquote><p><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">[<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;text&#8221;: &#8220;\u0421\u0456\u0301\u0440\u044b\u0443\u0441 (\u043b\u0430\u0446. Sirius), \u0442\u0430\u043a\u0441\u0430\u043c\u0430 \u0410\u043b\u044c\u0444\u0430 \u0412\u044f\u043b\u0456\u043a\u0430\u0433\u0430 \u0421\u0430\u0431\u0430\u043a\u0456 (\u043b\u0430\u0446. Alpha Canis Majoris) \u2013 \u043d\u0430\u0439\u044f\u0440\u0447\u044d\u0439\u0448\u0430\u044f \u0437\u043e\u0440\u043a\u0430 \u043d\u0430\u0447\u043d\u043e\u0433\u043e \u043d\u0435\u0431\u0430.&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;CharacterArr&#8221;:\u00a0<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&#8220;0&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;0&#8221;:\u00a0&#8220;\u0421\u0456&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;1&#8221;:\u00a0&#8220;alphabet&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;2&#8221;:\u00a02<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&#8220;1&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;0&#8221;: &#8220;\u0301&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;1&#8221;:\u00a0&#8220;character&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;2&#8221;:\u00a01<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&#8220;2&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;0&#8221;: &#8220;\u0440\u044b\u0443\u0441&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;1&#8221;:\u00a0&#8220;alphabet&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;2&#8221;:\u00a04<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&lt;&#8230;&gt;<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 },<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;ResultArr&#8221;:\u00a0{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;0&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;0&#8221;:\u00a0&#8220;\u0421\u0456\u0301\u0440\u044b\u0443\u0441&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;1&#8221;:\u00a0&#8220;word&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0&#8220;2&#8221;:\u00a07<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;1&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0\u00a0 \u00a0&#8220;0&#8221;: &#8221; &#8220;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0\u00a0 \u00a0&#8220;1&#8221;:\u00a0&#8220;whitespaces&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;2&#8221;:\u00a01<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0&#8220;2&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0\u00a0 \u00a0&#8220;0&#8221;: &#8220;(&#8220;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0\u00a0 \u00a0&#8220;1&#8221;:\u00a0&#8220;character&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0&#8220;2&#8221;: 1<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0&lt;&#8230;&gt;<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0\u00a0}<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0}<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">]<\/span><\/p><\/blockquote>\n<p>&nbsp;\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\"><img loading=\"lazy\" class=\"aligncenter size-full wp-image-6862\" src=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_EN_1.png\" alt=\"\" width=\"1015\" height=\"380\" srcset=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_EN_1.png 1015w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_EN_1-300x112.png 300w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_EN_1-768x288.png 768w\" sizes=\"(max-width: 1015px) 100vw, 1015px\" \/><\/a><\/p>\n<p style=\"text-align: center;\"><strong>Figure 1.<\/strong>\u00a0User interface of the service \u00abTokenizer\u00bb<\/p>\n<p>&nbsp;<\/p>\n<p><strong>The interface has the following areas:<\/strong><\/p>\n<p>&nbsp;<\/p>\n<ul>\n<li>text input field that requires tokenization;<\/li>\n<li>language selection menu;<\/li>\n<li>the button \u00abGet list of tokens!\u00bb, which starts tokenization;<\/li>\n<li>output field where selected tokens are displayed.<\/li>\n<\/ul>\n<p><strong>User script for working with the service<\/strong><\/p>\n<ol>\n<li>Enter the text that requires tokenization in the input field.<\/li>\n<li>Choose a language.<\/li>\n<li>Click the button \u00abGet list of tokens!\u00bb.<\/li>\n<li>View highlighted tokens (Figure 2).<\/li>\n<\/ol>\n<p>&nbsp;<\/p>\n<p style=\"text-align: center;\"><a href=\"http:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02.png\" class=\"grouped_elements\" rel=\"tc-fancybox-group5900\" title=\"\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\"><img loading=\"lazy\" class=\"wp-image-5902 aligncenter\" src=\"http:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02.png\" alt=\"\u041c\u0430\u043b\u044e\u043d\u0430\u043a 2. \u0412\u044b\u043d\u0456\u043a\u0456 \u043f\u0440\u0430\u0446\u044b \u0441\u044d\u0440\u0432\u0456\u0441\u0430 \u00ab\u0422\u0430\u043a\u0435\u043d\u0456\u0437\u0430\u0442\u0430\u0440\u00bb: \u0432\u044b\u043b\u0443\u0447\u0430\u043d\u044b\u044f \u0442\u043e\u043a\u0435\u043d\u044b\" width=\"700\" height=\"394\" srcset=\"https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02.png 801w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02-300x169.png 300w, https:\/\/ssrlab.by\/wp-content\/uploads\/2017\/08\/Tokenizer_screen02-768x432.png 768w\" sizes=\"(max-width: 700px) 100vw, 700px\" \/><\/a><strong>Figure 2.<\/strong>\u00a0\u00abTokenizer\u00bb results example<\/p>\n<p>&nbsp;<\/p>\n<p><strong>Access to the service via the API<\/strong><\/p>\n<p>To access the service\u00a0<strong><a href=\"https:\/\/corpus.by\/Tokenizer\/?lang=en\">\u00abTokenizer\u00bb<\/a><\/strong> via the API, you should send an AJAX-request (type: POST) to the address https:\/\/corpus.by\/Tokenizer\/api.php. With an input array\u00a0<em><strong>data\u00a0<\/strong><\/em>the following parameters are passed:<\/p>\n<ul>\n<li><strong><em>inputText<\/em><\/strong> \u2014 arbitrary input text.<\/li>\n<li><em><strong>language\u00a0<\/strong><\/em>\u2014 language of the input text. Belarusian (\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u00a0<strong><em>be<\/em><\/strong>), English (\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u00a0<strong><em>en<\/em><\/strong>) and Russian (\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u00a0<strong><em>ru<\/em><\/strong>) are available.<\/li>\n<\/ul>\n<p>Example of AJAX-request:<\/p>\n<blockquote><p><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">$.ajax({<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0type: &#8220;POST&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0url: &#8220;https:\/\/corpus.by\/Tokenizer\/api.php&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0data:{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;inputText&#8221;: &#8220;\u0421\u0456\u0301\u0440\u044b\u0443\u0441 (\u043b\u0430\u0446. Sirius), \u0442\u0430\u043a\u0441\u0430\u043c\u0430 \u0410\u043b\u044c\u0444\u0430 \u0412\u044f\u043b\u0456\u043a\u0430\u0433\u0430 \u0421\u0430\u0431\u0430\u043a\u0456 (\u043b\u0430\u0446. Alpha Canis Majoris) \u2013 \u043d\u0430\u0439\u044f\u0440\u0447\u044d\u0439\u0448\u0430\u044f \u0437\u043e\u0440\u043a\u0430 \u043d\u0430\u0447\u043d\u043e\u0433\u043e \u043d\u0435\u0431\u0430.&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;language&#8221;: &#8220;be&#8221;<\/span><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\"><br \/>\n},<br \/>\nsuccess: function(msg){ },<br \/>\nerror: function() { }<br \/>\n});<\/span><\/p><\/blockquote>\n<p>The server returns a JSON-array with the following parameters:<\/p>\n<ul>\n<li><em><strong>text\u00a0<\/strong><\/em>\u2014 input text.<\/li>\n<li><em><strong>CharacterArr<\/strong><\/em> \u2014 array with grouped character sequences.<\/li>\n<li><em><strong>ResultArr <\/strong><\/em>\u2014 array with character sequences grouped according to additional rules.<\/li>\n<\/ul>\n<p>For example, the following reply will be formed on the above listed AJAX-request:<\/p>\n<blockquote><p><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">[<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;text&#8221;: &#8220;\u0421\u0456\u0301\u0440\u044b\u0443\u0441 (\u043b\u0430\u0446. Sirius), \u0442\u0430\u043a\u0441\u0430\u043c\u0430 \u0410\u043b\u044c\u0444\u0430 \u0412\u044f\u043b\u0456\u043a\u0430\u0433\u0430 \u0421\u0430\u0431\u0430\u043a\u0456 (\u043b\u0430\u0446. Alpha Canis Majoris) \u2013 \u043d\u0430\u0439\u044f\u0440\u0447\u044d\u0439\u0448\u0430\u044f \u0437\u043e\u0440\u043a\u0430 \u043d\u0430\u0447\u043d\u043e\u0433\u043e \u043d\u0435\u0431\u0430.&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;CharacterArr&#8221;: <\/span><span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&#8220;0&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;0&#8221;:\u00a0&#8220;\u0421\u0456&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;1&#8221;:\u00a0&#8220;alphabet&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;2&#8221;:\u00a02<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&#8220;1&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;0&#8221;: &#8220;\u0301&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;1&#8221;:\u00a0&#8220;character&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;2&#8221;:\u00a01<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&#8220;2&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;0&#8221;: &#8220;\u0440\u044b\u0443\u0441&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;1&#8221;:\u00a0&#8220;alphabet&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 &#8220;2&#8221;:\u00a04<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 \u00a0 \u00a0&lt;&#8230;&gt;<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 },<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0 &#8220;ResultArr&#8221;:\u00a0{<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;0&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;0&#8221;:\u00a0&#8220;\u0421\u0456\u0301\u0440\u044b\u0443\u0441&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;1&#8221;:\u00a0&#8220;word&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0&#8220;2&#8221;:\u00a07<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;1&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0\u00a0 \u00a0&#8220;0&#8221;: &#8221; &#8220;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0\u00a0 \u00a0&#8220;1&#8221;:\u00a0&#8220;whitespaces&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0\u00a0 \u00a0&#8220;2&#8221;:\u00a01<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0&#8220;2&#8221;: {<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0\u00a0 \u00a0&#8220;0&#8221;: &#8220;(&#8220;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0\u00a0 \u00a0&#8220;1&#8221;:\u00a0&#8220;character&#8221;,<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0&#8220;2&#8221;: 1<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0 \u00a0 \u00a0\u00a0},<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0\u00a0\u00a0 \u00a0 \u00a0&lt;&#8230;&gt;<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0 \u00a0\u00a0}<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">\u00a0 \u00a0}<\/span><br \/>\n<span style=\"font-family: 'courier new', courier, monospace; font-size: 12px;\">]<\/span><\/p><\/blockquote>\n<p>&nbsp;<\/p>\n<p><strong>Links to sources<\/strong><\/p>\n<p>Service page:\u00a0<a href=\"https:\/\/corpus.by\/Tokenizer\/?lang=en\" target=\"_blank\" rel=\"noopener noreferrer\">https:\/\/corpus.by\/Tokenizer\/?lang=en<\/a><\/p>","protected":false},"excerpt":{"rendered":"<p class=\"qtranxs-available-languages-message qtranxs-available-languages-message-ru\">\u0418\u0437\u0432\u0438\u043d\u0438\u0442\u0435, \u044d\u0442\u043e\u0442 \u0442\u0435\u0445\u0442 \u0434\u043e\u0441\u0442\u0443\u043f\u0435\u043d \u0442\u043e\u043b\u044c\u043a\u043e \u0432 &ldquo;<a href=\"https:\/\/ssrlab.by\/BY\/wp-json\/wp\/v2\/posts\/5900\" class=\"qtranxs-available-language-link qtranxs-available-language-link-BY\" title=\"\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f\">\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f<\/a>&rdquo;. For the sake of viewer convenience, the content is shown below in the alternative language. You may click the link to switch the active language.<\/p>\n<p>\u0412\u044b\u043b\u0443\u0447\u0430\u0435 \u045e \u0442\u044d\u043a\u0441\u0446\u0435 \u045e\u0441\u0435 \u0442\u043e\u043a\u0435\u043d\u044b.<\/p>\n<a class = \"excerpt\" href=\"https:\/\/ssrlab.by\/ru\/5900\">(\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f) \u0427\u044b\u0442\u0430\u0446\u044c \u0431\u043e\u043b\u044c\u0448...<\/a>","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[5,292],"tags":[],"_links":{"self":[{"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/posts\/5900"}],"collection":[{"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/comments?post=5900"}],"version-history":[{"count":16,"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/posts\/5900\/revisions"}],"predecessor-version":[{"id":7473,"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/posts\/5900\/revisions\/7473"}],"wp:attachment":[{"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/media?parent=5900"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/categories?post=5900"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ssrlab.by\/ru\/wp-json\/wp\/v2\/tags?post=5900"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}