使用 NLPAUG 进行文本数据的扩充增强

pip install nlpaug
 conda install -c makecedward nlpaug

pip install nltk

from nlpaug.util.file.download import DownloadUtil
 DownloadUtil.download_word2vec(dest_dir='.') # Download word2vec model
 DownloadUtil.download_glove(model_name='glove.6B', dest_dir='.') # Download GloVe model
 DownloadUtil.download_fasttext(model_name='wiki-news-300d-1M', dest_dir='.') # Download fasttext model
 
 pip install gensim

pip install sacremoses

import nlpaug.augmenter.char as nac
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Keyboard
 aug = nac.KeyboardAug()
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.char as nac
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # OCR
 aug = nac.OcrAug()
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.char as nac
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Random - Character level - insert
 aug = nac.RandomCharAug(actinotallow="insert")
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.word as naw
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Synonym
 aug = naw.SynonymAug(aug_src='wordnet')
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.word as naw
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Reserved words
 reserved_tokens = ['lion', 'windows'] # Specify the reserved tokens here
 aug = naw.ReservedAug(reserved_tokens=reserved_tokens)
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.word as naw
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Contextual Word Embeddings - Word level
 aug = naw.ContextualWordEmbsAug()
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.word as naw
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Back translation
 aug = naw.BackTranslationAug()
 augmented_text = aug.augment(text)
 print(augmented_text)

import sklearn.datasets
 import re
 import nlpaug.augmenter.word as naw
 import nlpaug.model.word_stats as nmw
 
 # Defining a tokenizer function to extract word tokens
 def _tokenizer(text, token_pattern=r"(?u)\b\w\w+\b"):
    token_pattern = re.compile(token_pattern)
    return token_pattern.findall(text)
 
 # Load sample data (Scikit Learn 20 News Groups)
 train_data = sklearn.datasets.fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'))
 train_x = train_data.data
 
 # Tokenize input
 train_x_tokens = [_tokenizer(x) for x in train_x]
 
 # Train TF-IDF model
 tfidf_model = nmw.TfIdf()
 tfidf_model.train(train_x_tokens)
 tfidf_model.save('.')
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Augment the text with TFIDF augmenter
 aug = naw.TfIdfAug(model_path='.', tokenizer=_tokenizer)
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.sentence as nas
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Abstractive Summarization
 aug = nas.AbstSummAug()
 augmented_text = aug.augment(text)
 print(augmented_text)

import nlpaug.augmenter.sentence as nas
 
 text = "It was a dark and stormy night. I was alone at home when I saw a lion's face followed by a scary thunderous roar at the windows."
 
 # Contextual Word Embeddings - Sentence level
 aug = nas.ContextualWordEmbsForSentenceAug()
 augmented_text = aug.augment(text)
 print(augmented_text)