File size: 10,494 Bytes
8453337 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | import argparse
import sys
sys.path.append('../')
from time import perf_counter
from lyra_llama import lyraLlama
def get_args():
parser = argparse.ArgumentParser(description="Faster ChatGLM6B Demo")
parser.add_argument('--model-path', type=str, required=True,
help='Model Path, include config.ini and tokenizer files')
parser.add_argument('--tokenizer-path', type=str, default=None)
parser.add_argument(
'--data-type', type=str, metavar='TYPE', default='fp16',
choices=[None, 'fp32', 'fp16', 'bf16', 'int8'],
help='The data type to inference. If None, the data type follows the '
'checkpoint data type.')
parser.add_argument(
'--memopt-mode', type=int, default=0, choices=[0, 1],
help='Use MEMOPT mode to increase speed and reduce VRAM usage.'
' 0: FP16 mode'
' 1: Use MEMOPT mode')
parser.add_argument(
'--quant-type', type=str, metavar='TYPE', default='int8',
choices=['int4', 'int8'],
help='The data type of quantization. Only used in MEMOPT.')
parser.add_argument(
'--kvqparams-fpath', type=str, required=False, default="",
help='File path of kv quantized params.')
parser.add_argument("--prompt", type=str, required=False)
parser.add_argument("--max-output-length", type=int, default=512)
parser.add_argument("--warmups", type=int, default=10)
parser.add_argument("--avgnums", type=int, default=10)
args = parser.parse_args()
print('\n=================== Arguments ===================')
for k, v in vars(args).items():
print(f' - {k.ljust(25, ".")}: {v}')
print('=================================================')
return args
def main():
args = get_args()
model = lyraLlama(args.model_path, args.tokenizer_path, args.data_type, args.memopt_mode, args.quant_type, args.kvqparams_fpath)
# args.prompt = '''<context>/*\n * Implement the \"Falling Rocks\" game in the text console. \n * A small dwarf stays at the bottom of the screen and can \n * move left and right (by the arrows keys). A number of rocks \n * of different sizes and forms constantly fall down and you \n * need to avoid a crash.\n * Rocks are the symbols ^, @, *, &, +, %, $, #, !, ., ;, - distributed \n * with appropriate density. The dwarf is (O). \n * Ensure a constant game speed by Thread.Sleep(150).\n * Implement collision detection and scoring system.\n*/\n\nusing System;\nusing System.Threading;\nusing System.Collections.Generic;\nusing System.Threading.Tasks;\n\nclass FallingRocks\n{\n struct Position\n {\n public int X, Y;\n public string symbol;\n public ConsoleColor color;\n\n public Position(int x, int y, string symbol, ConsoleColor color)\n {\n this.X = x;\n this.Y = y;\n this.symbol = symbol;\n this.color = color;\n }\n }\n\n static void Main()\n {\n Thread oThread = new Thread(new ThreadStart(Mainn));\n Thread aThread = new Thread(new ThreadStart(Clr));\n \n aThread.Start();\n oThread.Start();\n oThread.Join();\n aThread.Join();\n }\n\n static void Clr()\n {\n while (true)\n {\n Thread.Sleep(10);\n Console.Clear();\n }\n }\n static void Mainn()\n {\n //Random generator for rocks color, position and symbol\n Random randomGenerator = new Random();\n \n //Sleep time for the game loop\n double sleepTime = 150;\n //Console settings\n Console.CursorVisible = false;\n Console.BufferHeight = Console.WindowHeight;\n \n //number of rocks in the Array rocks\n int rocksCount = 0;\n\n //array with the symbols of the rocks\n string[] symbols = new string[] { \"^\", \"@\", \"*\", \"&\", \"+\", \"%\", \"$\", \"#\", \"!\", \".\", \";\" };\n \n //array with colors for the rocks\n ConsoleColor[] colors = new ConsoleColor[] {ConsoleColor.Yellow, ConsoleColor.White, ConsoleColor.Gray};\n \n //array with rocks\n Position[] rocks = new Position[200];\n \n //position for the dwarf\n Position dwarf = new Position(10, Console.WindowHeight - 1,\"(0)\",ConsoleColor.Red);\n \n //bool variable to say when the game loop to be over\n bool gameLoop = true;\n\n //variable keeping the score\n ulong score = 0;\n\n //the game loop\n while (gameLoop)\n {\n //score is growing as the cycle runs\n score++;\n\n //setting the Y component for all the rocks in the array to grow with 2\n for (int i = 0; i <= rocks.Length - 1; i++)\n {\n rocks[i].Y = rocks[i].Y + 2;\n }\n\n //generating rocks\n for (int x = 0; x <= randomGenerator.Next(2, 4); x++)\n {\n rocks[rocksCount] = new Position(randomGenerator.Next(x * 15, x * 15 + 20), 0\n , symbols[randomGenerator.Next(0, symbols.Length - 1)]\n , colors[randomGenerator.Next(0, colors.Length - 1)]);\n if (rocksCount >= 199) rocksCount = 0;\n rocksCount++;\n }\n\n //printing the rocks and other stuff\n foreach (var item in rocks)\n {\n foreach (var rock in rocks)\n {\n //checking for colision\n if ((rock.X >= dwarf.X) && (rock.X <= (dwarf.X + 2)) && (rock.Y == dwarf.Y))\n {\n gameLoop = false;\n break;\n }\n } \n\n //printing the rocks\n if (item.Y < Console.WindowHeight)\n { \n Console.SetCursorPosition(item.X, item.Y);\n Console.ForegroundColor = item.color;\n Console.Write(item.symbol);\n }\n\n //checking for key pressed\n if (Console.KeyAvailable)\n {\n ConsoleKeyInfo pressedKey = Console.ReadKey();\n if (pressedKey.Key == ConsoleKey.RightArrow)\n {\n if(dwarf.X < Console.WindowWidth - 20)\n {\n //removing the old positions of the dwarf and increasing his X value\n Console.SetCursorPosition(dwarf.X, dwarf.Y);\n Console.Write(\" \");\n dwarf.X++;\n }\n }\n if (pressedKey.Key == ConsoleKey.LeftArrow) \n {\n if(dwarf.X >= 1)\n {\n //removing the old positions of the dwarf and decreasing his X value\n Console.SetCursorPosition(dwarf.X, dwarf.Y);\n Console.Write(\" \");\n dwarf.X--;\n }\n }\n }\n }\n \n //printing the dwarf\n Console.SetCursorPosition(dwarf.X, dwarf.Y);\n Console.ForegroundColor = dwarf.color;\n Console.Write(dwarf.symbol); \n \n //sleeping the loop for sometime\n //Thread.Sleep((int)sleepTime);\n\n //reducing the sleep time of the loop\n sleepTime -= 0.5;\n\n \n //removing the rocks \n //foreach (var item in rocks)\n //{\n // if (item.Y < Console.WindowHeight)\n // {\n // Console.SetCursorPosition(item.X, item.Y);\n // Console.Write(\" \");\n // }\n //} \n }\n //Printing the score after the game is over\n Console.Clear();\n Console.WriteLine(\"Game over! Your score is: \" + score);\n\n }\n}\n</context>\n\n这个\"Falling Rocks\"游戏是如何工作的呢？可以详细解释一下代码的运作机制吗？ \n\n\n\n'''
prompt_template = "Human: {}\n\nAssistant:" # xverse
# prompt_template = "<human>:{}\n<bot>:" # llama-ziya 13b
prompt = prompt_template.format(args.prompt)
test_batch_size = [1, 8, 16, 32, 64] # 8, 16, 32, 64
print("test_batch_size: ", test_batch_size)
for i, bs in enumerate(test_batch_size):
prompts = [prompt, ] * bs
# warmup gpu
for _ in range(args.warmups):
output_texts = model.generate(
prompts, output_length=args.max_output_length,
top_k=30, top_p=0.85, temperature=1.0, repetition_penalty=1.0, do_sample=False)
start = perf_counter()
for _ in range(args.avgnums):
output_texts = model.generate(
prompts, output_length=args.max_output_length,
top_k=30, top_p=0.85, temperature=1.0, repetition_penalty=1.0, do_sample=False)
end = perf_counter()
cost = (end - start) / args.avgnums
input_output_texts = [prompt + ' ' + gtext for prompt,
gtext in zip(prompts, output_texts)]
tokens = 0
input_tokens = len(model.tokenizer.encode(prompt))
words = 0
for text in input_output_texts:
tokens += len(model.tokenizer.encode(text))
words += len(text)
avg_output_tokens = tokens / len(input_output_texts) - input_tokens
print(
f"\nFaster-Dtype: {args.data_type}, Batch Size: {bs}, All tokens: {tokens}. Input tokens: {input_tokens}. Output tokens: {avg_output_tokens} Cost: {cost} seconds. Speed: {tokens/cost} tokens/s."
)
print(
f"Faster-Dtype: {args.data_type}, Batch Size: {bs}, All generated words: {words}. Cost: {cost} seconds. Speed: {words/cost} words/s."
)
if i == 0:
for k in range(bs):
print(
f"The {k} Sample, \n\t\tInputs: {prompts[k]}. \n\t\tOutputs: {output_texts[k].lstrip()}")
if k > 2:
break
if __name__ == "__main__":
main()
|