怎样建设大型网站,电子商务网站建设作文,设计软件图标,企业免费建网站本文主要介绍了Rasa中相关Tokenizer的具体实现#xff0c;包括默认Tokenizer和第三方Tokenizer。前者包括JiebaTokenizer、MitieTokenizer、SpacyTokenizer和WhitespaceTokenizer#xff0c;后者包括BertTokenizer和AnotherWhitespaceTokenizer。
一.JiebaTokenizer Ji… 本文主要介绍了Rasa中相关Tokenizer的具体实现包括默认Tokenizer和第三方Tokenizer。前者包括JiebaTokenizer、MitieTokenizer、SpacyTokenizer和WhitespaceTokenizer后者包括BertTokenizer和AnotherWhitespaceTokenizer。
一.JiebaTokenizer JiebaTokenizer类整体代码结构如下所示 加载自定义字典代码如下所示[3]
staticmethod
def _load_custom_dictionary(path: Text) - None:Load all the custom dictionaries stored in the path. # 加载存储在路径中的所有自定义字典。More information about the dictionaries file format can be found in the documentation of jieba. https://github.com/fxsjy/jieba#load-dictionaryprint(JiebaTokenizer._load_custom_dictionary())import jiebajieba_userdicts glob.glob(f{path}/*) # 获取路径下的所有文件。for jieba_userdict in jieba_userdicts: # 遍历所有文件。logger.info(fLoading Jieba User Dictionary at {jieba_userdict}) # 加载结巴用户字典。jieba.load_userdict(jieba_userdict) # 加载用户字典。实现分词的代码为tokenize()方法如下所示
def tokenize(self, message: Message, attribute: Text) - List[Token]:Tokenizes the text of the provided attribute of the incoming message. # 对传入消息的提供属性的文本进行tokenize。print(JiebaTokenizer.tokenize())import jiebatext message.get(attribute) # 获取消息的属性tokenized jieba.tokenize(text) # 对文本进行标记化tokens [Token(word, start) for (word, start, end) in tokenized] # 生成标记return self._apply_token_pattern(tokens)self._apply_token_pattern(tokens)数据类型为List[Token]。Token的数据类型为
class Token:# 由将单个消息拆分为多个Token的Tokenizers使用def __init__(self,text: Text,start: int,end: Optional[int] None,data: Optional[Dict[Text, Any]] None,lemma: Optional[Text] None,) - None:创建一个TokenArgs:text: The token text. # token文本start: The start index of the token within the entire message. # token在整个消息中的起始索引end: The end index of the token within the entire message. # token在整个消息中的结束索引data: Additional token data. # 附加的token数据lemma: An optional lemmatized version of the token text. # token文本的可选词形还原版本self.text textself.start startself.end end if end else start len(text)self.data data if data else {}self.lemma lemma or text特别说明JiebaTokenizer组件的is_trainableTrue。 二.MitieTokenizer MitieTokenizer类整体代码结构如下所示 核心代码tokenize()方法代码如下所示
def tokenize(self, message: Message, attribute: Text) - List[Token]:Tokenizes the text of the provided attribute of the incoming message. # 对传入消息的提供属性的文本进行tokenizeimport mitietext message.get(attribute)encoded_sentence text.encode(DEFAULT_ENCODING)tokenized mitie.tokenize_with_offsets(encoded_sentence)tokens [self._token_from_offset(token, offset, encoded_sentence)for token, offset in tokenized]return self._apply_token_pattern(tokens)特别说明mitie库在Windows上安装可能麻烦些。MitieTokenizer组件的is_trainableFalse。 三.SpacyTokenizer 首先安装Spacy类库和模型[4][5]如下所示
pip3 install -U spacy
python3 -m spacy download zh_core_web_smSpacyTokenizer类整体代码结构如下所示 核心代码tokenize()方法代码如下所示
def tokenize(self, message: Message, attribute: Text) - List[Token]:Tokenizes the text of the provided attribute of the incoming message. # 对传入消息的提供属性的文本进行tokenizedoc self._get_doc(message, attribute) # doc是一个Doc对象if not doc:return []tokens [Token(t.text, t.idx, lemmat.lemma_, data{POS_TAG_KEY: self._tag_of_token(t)})for t in docif t.text and t.text.strip()]特别说明SpacyTokenizer组件的is_trainableFalse。即SpacyTokenizer只有运行组件run_SpacyTokenizer0没有训练组件。如下所示
四.WhitespaceTokenizer WhitespaceTokenizer主要是针对英文的不可用于中文。WhitespaceTokenizer类整体代码结构如下所示 其中predict_schema和train_schema如下所示 rasa shell nlu --debug结果如下所示 特别说明WhitespaceTokenizer组件的is_trainableFalse。 五.BertTokenizer rasa shell nlu --debug结果如下所示
emsp;emsp;BertTokenizer代码具体实现如下所示https://github.com/daiyizheng/rasa-chinese-plus/blob/master/rasa_chinese_plus/nlu/tokenizers/bert_tokenizer.pyfrom typing import List, Text, Dict, Any
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
from rasa.shared.nlu.training_data.message import Message
from transformers import AutoTokenizer
from rasa.nlu.tokenizers.tokenizer import Tokenizer, TokenDefaultV1Recipe.register(DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainableFalse
)
class BertTokenizer(Tokenizer):def __init__(self, config: Dict[Text, Any] None) - None::param config: {pretrained_model_name_or_path:, cache_dir:, use_fast:}super().__init__(config)self.tokenizer AutoTokenizer.from_pretrained(config[pretrained_model_name_or_path], # 指定预训练模型的名称或路径cache_dirconfig.get(cache_dir), # 指定缓存目录use_fastTrue if config.get(use_fast) else False # 是否使用快速模式)classmethoddef required_packages(cls) - List[Text]:return [transformers] # 指定依赖的包staticmethoddef get_default_config() - Dict[Text, Any]:The components default config (see parent class for full docstring).return {# Flag to check whether to split intentsintent_tokenization_flag: False,# Symbol on which intent should be splitintent_split_symbol: _,# Regular expression to detect tokenstoken_pattern: None,# Symbol on which prefix should be splitprefix_separator_symbol: None,}def tokenize(self, message: Message, attribute: Text) - List[Token]:text message.get(attribute) # 获取文本encoded_input self.tokenizer(text, return_offsets_mappingTrue, add_special_tokensFalse) # 编码文本token_position_pair zip(encoded_input.tokens(), encoded_input[offset_mapping]) # 将编码后的文本和偏移量映射成一个元组tokens [Token(texttoken_text, startposition[0], endposition[1]) for token_text, position in token_position_pair] # 将元组转换成Token对象return self._apply_token_pattern(tokens)特别说明BertTokenizer组件的is_trainableFalse。 六.AnotherWhitespaceTokenizer AnotherWhitespaceTokenizer代码具体实现如下所示
from __future__ import annotations
from typing import Any, Dict, List, Optional, Textfrom rasa.engine.graph import ExecutionContext
from rasa.engine.recipes.default_recipe import DefaultV1Recipe
from rasa.engine.storage.resource import Resource
from rasa.engine.storage.storage import ModelStorage
from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
from rasa.shared.nlu.training_data.message import MessageDefaultV1Recipe.register(DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainableFalse
)
class AnotherWhitespaceTokenizer(Tokenizer):Creates features for entity extraction.staticmethoddef not_supported_languages() - Optional[List[Text]]:The languages that are not supported.return [zh, ja, th]staticmethoddef get_default_config() - Dict[Text, Any]:Returns the components default config.return {# This *must* be added due to the parent class.intent_tokenization_flag: False,# This *must* be added due to the parent class.intent_split_symbol: _,# This is a, somewhat silly, config that we passonly_alphanum: True,}def __init__(self, config: Dict[Text, Any]) - None:Initialize the tokenizer.super().__init__(config)self.only_alphanum config[only_alphanum]def parse_string(self, s):if self.only_alphanum:return .join([c for c in s if ((c ) or str.isalnum(c))])return sclassmethoddef create(cls,config: Dict[Text, Any],model_storage: ModelStorage,resource: Resource,execution_context: ExecutionContext,) - AnotherWhitespaceTokenizer:return cls(config)def tokenize(self, message: Message, attribute: Text) - List[Token]:text self.parse_string(message.get(attribute))words [w for w in text.split( ) if w]# if we removed everything like smiles :), use the whole text as 1 tokenif not words:words [text]# the ._convert_words_to_tokens() method is from the parent class.tokens self._convert_words_to_tokens(words, text)return self._apply_token_pattern(tokens)特别说明AnotherWhitespaceTokenizer组件的is_trainableFalse。 参考文献 [1]自定义Graph Component1.1-JiebaTokenizer具体实现https://mp.weixin.qq.com/s/awGiGn3uJaNcvJBpk4okCA [2]https://github.com/RasaHQ/rasa [3]https://github.com/fxsjy/jieba#load-dictionary [4]spaCy GitHubhttps://github.com/explosion/spaCy [5]spaCy官网https://spacy.io/ [6]https://github.com/daiyizheng/rasa-chinese-plus/blob/master/rasa_chinese_plus/nlu/tokenizers/bert_tokenizer.py
