Update pipeline.py
Browse files- pipeline.py +2 -0
pipeline.py
CHANGED
@@ -2,6 +2,7 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
|
2 |
from typing import Dict, List, Any
|
3 |
import itertools
|
4 |
from nltk import sent_tokenize
|
|
|
5 |
|
6 |
class PreTrainedPipeline():
|
7 |
def __init__(self, path=""):
|
@@ -9,6 +10,7 @@ class PreTrainedPipeline():
|
|
9 |
# Preload all the elements you are going to need at inference.
|
10 |
# For instance your model, processors, tokenizer that might be needed.
|
11 |
# This function is only called once, so do all the heavy processing I/O here"""
|
|
|
12 |
self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
|
13 |
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
14 |
|
|
|
2 |
from typing import Dict, List, Any
|
3 |
import itertools
|
4 |
from nltk import sent_tokenize
|
5 |
+
import nltk
|
6 |
|
7 |
class PreTrainedPipeline():
|
8 |
def __init__(self, path=""):
|
|
|
10 |
# Preload all the elements you are going to need at inference.
|
11 |
# For instance your model, processors, tokenizer that might be needed.
|
12 |
# This function is only called once, so do all the heavy processing I/O here"""
|
13 |
+
nltk.download('punkt')
|
14 |
self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
|
15 |
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
16 |
|