Original model card

Buy me a coffee if you like this project ;) <a href="https://www.buymeacoffee.com/s3nh"><img src="https://www.buymeacoffee.com/assets/img/guidelines/download-assets-sm-1.svg" alt=""></a>

Description

GGML Format model files for This project.

inference

import ctransformers

from ctransformers import AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(output_dir, ggml_file,
gpu_layers=32, model_type="llama")

manual_input: str = "Tell me about your last dream, please."


llm(manual_input, 
      max_new_tokens=256, 
      temperature=0.9, 
      top_p= 0.7)

Original model card

from transformers import LlamaForCausalLM, LlamaTokenizer
import torch

def generate_prompt(query, history, input=None):
    prompt = ""
    for i, (old_query, response) in enumerate(history):
        prompt += "{}{}\n<end>".format(old_query, response)
    prompt += "{}".format(query)
    return prompt



# Load model
device = torch.device("cuda:0")
model_name = "/data/dell/xuyipei/my_llama/my_llama_13b/llama_13b_112_sft_v1"
tokenizer = LlamaTokenizer.from_pretrained(model_name)
model = LlamaForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.float16
)
model.eval()
model = model.to(device)


# Inference
history = []
queries = ['请推荐五本名著，依次列出作品名、作者\n', '请再来三本\n']
memory_limit = 3 # the number of (query, response) to remember
for query in queries:
    prompt = generate_prompt(prompt, history)
    input_ids = tokenizer(query, return_tensors="pt", padding=False, truncation=False, add_special_tokens=False)
    input_ids = input_ids["input_ids"].to(device)

    with torch.no_grad():
        outputs=model.generate(
                input_ids=input_ids,
                top_p=0.8,
                top_k=50,
                repetition_penalty=1.1,
                max_new_tokens = 256,
                early_stopping = True,
                eos_token_id = tokenizer.convert_tokens_to_ids('<end>'),
                pad_token_id = tokenizer.eos_token_id,
                min_length = input_ids.shape[1] + 1
        )
    s = outputs[0]
    response=tokenizer.decode(s)
    response = response.replace('<s>', '').replace('<end>', '').replace('</s>', '')
    print(response)
    history.append((query, response))
    history = history[-memory_limit:]