File size: 864 Bytes
1d5604f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
#!/usr/bin/env python3
# coding=utf-8
import utility.parser_utils as utils
from data.parser.from_mrp.abstract_parser import AbstractParser
class RequestParser(AbstractParser):
def __init__(self, sentences, args, fields):
self.data = {i: {"id": str(i), "sentence": sentence} for i, sentence in enumerate(sentences)}
sentences = [example["sentence"] for example in self.data.values()]
for example in self.data.values():
example["input"] = example["sentence"].strip().split(' ')
example["token anchors"], offset = [], 0
for token in example["input"]:
example["token anchors"].append([offset, offset + len(token)])
offset += len(token) + 1
utils.create_bert_tokens(self.data, args.encoder)
super(RequestParser, self).__init__(fields, self.data)
|