语言模型数据处理

data = [("me gusta comer en la cafeteria".split(), "SPANISH"),
        ("Give it to me".split(), "ENGLISH"),
        ("No creo que sea una buena idea".split(), "SPANISH"),
        ("No it is not a good idea to get lost at sea".split(), "ENGLISH")]

out[]

[([‘me’, ‘gusta’, ‘comer’, ‘en’, ‘la’, ‘cafeteria’], ‘SPANISH’),
([‘Give’, ‘it’, ‘to’, ‘me’], ‘ENGLISH’),
([‘No’, ‘creo’, ‘que’, ‘sea’, ‘una’, ‘buena’, ‘idea’], ‘SPANISH’),
([‘No’, ‘it’, ‘is’, ‘not’, ‘a’, ‘good’, ‘idea’, ‘to’, ‘get’, ‘lost’, ‘at’, ‘sea’], ‘ENGLISH’)]

word_to_ix = {}
for sent, _ in data + test_data:
    for word in sent:
        if word not in word_to_ix:
            word_to_ix[word] = len(word_to_ix)
print(word_to_ix)
print (word)

out[] sent

[‘me’, ‘gusta’, ‘comer’, ‘en’, ‘la’, ‘cafeteria’]
[‘Give’, ‘it’, ‘to’, ‘me’]
[‘No’, ‘creo’, ‘que’, ‘sea’, ‘una’, ‘buena’, ‘idea’]
[‘No’, ‘it’, ‘is’, ‘not’, ‘a’, ‘good’, ‘idea’, ‘to’, ‘get’, ‘lost’, ‘at’, ‘sea’]

out[] word

me
gusta
comer
en
la
cafeteria
Give

……

out[] word_to_ix

{‘en’: 3, ‘No’: 9, ‘buena’: 14, ‘it’: 7, ‘at’: 22, ‘sea’: 12, ‘cafeteria’: 5, ‘la’: 4, ‘to’: 8, ‘creo’: 10, ‘is’: 16, ‘a’: 18, ‘good’: 19, ‘get’: 20, ‘idea’: 15, ‘que’: 11, ‘not’: 17, ‘me’: 0, ‘gusta’: 1, ‘lost’: 21, ‘Give’: 6, ‘una’: 13, ‘comer’: 2}

import os
import torch

from collections import Counter


class Dictionary(object):
    def __init__(self):
        self.word2idx = {}
        self.idx2word = []
        self.counter = Counter()
        self.total = 0

    def add_word(self, word):
        if word not in self.word2idx:
            self.idx2word.append(word)
            self.word2idx[word] = len(self.idx2word) - 1
        token_id = self.word2idx[word]
        self.counter[token_id] += 1
        self.total += 1
        return self.word2idx[word]

    def __len__(self):
        return len(self.idx2word)


class Corpus(object):
    def __init__(self, path):
        self.dictionary = Dictionary()
        self.train = self.tokenize(os.path.join(path, 'train.txt'))
        self.valid = self.tokenize(os.path.join(path, 'valid.txt'))
        self.test = self.tokenize(os.path.join(path, 'test.txt'))

    def tokenize(self, path):
        """Tokenizes a text file."""
        assert os.path.exists(path)
        # Add words to the dictionary
        with open(path, 'r') as f:
            tokens = 0
            for line in f:
                words = line.split() + ['<eos>']
                tokens += len(words)
                for word in words:
                    self.dictionary.add_word(word)

        # Tokenize file content
        with open(path, 'r') as f:
            ids = torch.LongTensor(tokens)
            token = 0
            for line in f:
                words = line.split() + ['<eos>']
                for word in words:
                    ids[token] = self.dictionary.word2idx[word]
                    token += 1

        return ids