cahce

noahshinn · Aug 28, 2023 · be7fb52 · be7fb52
1 parent 3fec014
commit be7fb52
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/programming_runs/generators/model.py b/programming_runs/generators/model.py
@@ -132,6 +132,7 @@ def generate_chat(self, messages: List[Message], max_tokens: int = 1024, tempera
             prompt,
             max_new_tokens=min(
                 max_tokens, self.model.config.max_position_embeddings),
+            use_cache=True,
             do_sample=True,
             temperature=temperature,
             top_p=0.95,