togethercomputer
/

RedPajama-INCITE-7B-Chat

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

zhangce commited on May 5, 2023

Commit

9c5de29

•

1 Parent(s): 6f2c961

Update README.md

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -28,7 +28,7 @@ tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-Chat-INCIT
 model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Chat-INCITE-6.9B-v1", torch_dtype=torch.float16)
 model = model.to('cuda:0')
 # infer
-inputs = tokenizer("Hello", return_tensors='pt').to(model.device)
 outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
 output_str = tokenizer.decode(outputs[0])
 print(output_str)
@@ -44,7 +44,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-Chat-INCITE-6.9B-v1")
 model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Chat-INCITE-6.9B-v1", device_map="auto", load_in_8bit=True)
 # infer
-inputs = tokenizer("Hello", return_tensors='pt').to(model.device)
 outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
 output_str = tokenizer.decode(outputs[0])
 print(output_str)

 model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Chat-INCITE-6.9B-v1", torch_dtype=torch.float16)
 model = model.to('cuda:0')
 # infer
+inputs = tokenizer("<human>: Hello!\n<bot>:", return_tensors='pt').to(model.device)
 outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
 output_str = tokenizer.decode(outputs[0])
 print(output_str)
 tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-Chat-INCITE-6.9B-v1")
 model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-Chat-INCITE-6.9B-v1", device_map="auto", load_in_8bit=True)
 # infer
+inputs = tokenizer("<human>: Hello!\n<bot>:", return_tensors='pt').to(model.device)
 outputs = model.generate(**inputs, max_new_tokens=10, do_sample=True, temperature=0.8)
 output_str = tokenizer.decode(outputs[0])
 print(output_str)