Skip to content

Commit 9568553

Browse files
tybalextybalex
committed
Hacky func streaming (#1)
* hacky function call streaming * remove * minor fix to take care of case that the input function has no description or arguments is null * test parser * fix makefile to make sure the order of file linking works for ubuntu gcc/g++ 11.4 * add function name mapping to take care of input function name with hyphen- * add a comment TODO for streaming chunks.
1 parent 594fe54 commit 9568553

File tree

1 file changed

+90
-0
lines changed

1 file changed

+90
-0
lines changed

‎test_llamacpp.ipynb

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@
141141
" 'required': []\n",
142142
" }\n",
143143
" }\n",
144+
<<<<<<< HEAD
144145
=======
145146
" print(e, model, prompt)\n"
146147
]
@@ -172,6 +173,8 @@
172173
" \"location\": {\"type\": \"string\", \"description\": \"The city and state e.g. San Francisco, CA\"},\n",
173174
" \"unit\": {\"type\": \"string\", \"enum\": [\"c\", \"f\"]}\n",
174175
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
176+
=======
177+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
175178
" },\n",
176179
" {\n",
177180
" 'type': 'function',\n",
@@ -314,6 +317,9 @@
314317
{
315318
"cell_type": "code",
316319
<<<<<<< HEAD
320+
<<<<<<< HEAD
321+
=======
322+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
317323
"execution_count": 4,
318324
"metadata": {},
319325
"outputs": [],
@@ -326,9 +332,12 @@
326332
{
327333
"cell_type": "code",
328334
"execution_count": 5,
335+
<<<<<<< HEAD
329336
=======
330337
"execution_count": 177,
331338
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
339+
=======
340+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
332341
"metadata": {},
333342
"outputs": [
334343
{
@@ -348,22 +357,29 @@
348357
"\n",
349358
"[AI response]:\n",
350359
" The distance between San Francisco and Cupertino by driving is 50 miles and 100 miles from Cupertino to San Francisco. When traveling by air, the distance is 150 miles from San Francisco to Cupertino and 200 miles from Cupertino to San Francisco.\n"
360+
<<<<<<< HEAD
351361
=======
352362
" <<functions>>[get_stock_fundermentals(symbol=\"TSLA\")]\n",
353363
"<<functions>>[get_stock_fundermentals(symbol=\"GOOG\")]\n"
354364
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
365+
=======
366+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
355367
]
356368
}
357369
],
358370
"source": [
359371
<<<<<<< HEAD
372+
<<<<<<< HEAD
373+
=======
374+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
360375
"\n",
361376
"get_mistral_rubra_response = partial(get_oai_response, api_key=local_api_key, base_url=local_base_url)\n",
362377
"\n",
363378
"user_query = \"What is the distance between San Francisco and Cupertino by driving and by air from both directions?\"\n",
364379
"# user_query = \"What is four plus six? What is the result of that plus 2? Take the result and multiply by 5 and then divide by two\"\n",
365380
"# user_query = \"what's the distance between SF and NYC? Use the result value to multiply by 8, and then divide by 2, and then minus 30\"\n",
366381
"msgs = run_completion(get_mistral_rubra_response, user_query)\n"
382+
<<<<<<< HEAD
367383
=======
368384
"system_prompt = \"You are a helpful assistant.\"\n",
369385
"functions = [\n",
@@ -415,21 +431,62 @@
415431
"res = get_mistral_rubra_response(user_query, \"mistral_rubra\", functions=functions, msgs=msgs)\n",
416432
"print(res.message.content)"
417433
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
434+
=======
435+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
418436
]
419437
},
420438
{
421439
"cell_type": "code",
440+
<<<<<<< HEAD
422441
<<<<<<< HEAD
423442
"execution_count": 8,
424443
=======
425444
"execution_count": 183,
426445
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
446+
=======
447+
"execution_count": 8,
448+
"metadata": {},
449+
"outputs": [
450+
{
451+
"name": "stdout",
452+
"output_type": "stream",
453+
"text": [
454+
"Pointing to URL: http://localhost:8019/v1/\n",
455+
"\n",
456+
"[AI calling functions]:\n",
457+
"Tool Call: Function(arguments='{\"number_to_buy\":3}', name='orderUmbrella')\n",
458+
"Tool Call: Function(arguments='{\"length\":8}', name='generate_password')\n",
459+
"Pointing to URL: http://localhost:8019/v1/\n",
460+
"Loop 0\n",
461+
"\n",
462+
"[AI response]:\n",
463+
" Your order for 3 umbrellas has been placed, and the total price is 10 dollars. The generated password is 96ddefe8.\n"
464+
]
465+
}
466+
],
467+
"source": [
468+
"user_query2 = \"now order 3 umbrellas for me and generate a password of length 8\"\n",
469+
"msgs = run_completion(get_mistral_rubra_response, user_query2, msgs)"
470+
]
471+
},
472+
{
473+
"cell_type": "markdown",
474+
"metadata": {},
475+
"source": [
476+
"## Simple Math Chaining"
477+
]
478+
},
479+
{
480+
"cell_type": "code",
481+
"execution_count": 9,
482+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
427483
"metadata": {},
428484
"outputs": [
429485
{
430486
"name": "stdout",
431487
"output_type": "stream",
432488
"text": [
489+
<<<<<<< HEAD
433490
<<<<<<< HEAD
434491
"Pointing to URL: http://localhost:8019/v1/\n",
435492
"\n",
@@ -445,18 +502,47 @@
445502
"<ast.List object at 0x105649390>\n",
446503
"[('get_current_weather', [], {'location': 'Boston, MA', 'api_key': 123456789, 'unit': 'fahrenheit'}), ('func', ['cde'], {'x': 1, 'b': '2', 'c': [1, 2, {'a': 1, 'b': 2}]})]\n"
447504
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
505+
=======
506+
"Pointing to URL: http://localhost:8019/v1/\n",
507+
"\n",
508+
"[AI calling functions]:\n",
509+
"Tool Call: Function(arguments='{\"a\":\"4\",\"b\":\"6\"}', name='addition')\n",
510+
"Tool Call: Function(arguments='{\"a\":\"result\",\"b\":\"2\"}', name='addition')\n",
511+
"Tool Call: Function(arguments='{\"a\":\"result\",\"b\":\"5\"}', name='multiplication')\n",
512+
"Tool Call: Function(arguments='{\"a\":\"result\",\"b\":\"2\"}', name='division')\n"
513+
]
514+
},
515+
{
516+
"ename": "ValueError",
517+
"evalue": "could not convert string to float: 'result'",
518+
"output_type": "error",
519+
"traceback": [
520+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
521+
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
522+
"Cell \u001b[0;32mIn[9], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m user_query3 \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUser tool to help me : What is four plus six? What is the result of that plus 2? Take the result and multiply by 5 and then divide by two\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m----> 2\u001b[0m msgs \u001b[38;5;241m=\u001b[39m \u001b[43mrun_completion\u001b[49m\u001b[43m(\u001b[49m\u001b[43mget_mistral_rubra_response\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muser_query3\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsgs\u001b[49m\u001b[43m)\u001b[49m\n",
523+
"Cell \u001b[0;32mIn[3], line 244\u001b[0m, in \u001b[0;36mrun_completion\u001b[0;34m(chat_method, user_query, msgs)\u001b[0m\n\u001b[1;32m 242\u001b[0m l \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 243\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m res_next\u001b[38;5;241m.\u001b[39mmessage\u001b[38;5;241m.\u001b[39mtool_calls \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(res_next\u001b[38;5;241m.\u001b[39mmessage\u001b[38;5;241m.\u001b[39mtool_calls) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 244\u001b[0m msgs \u001b[38;5;241m=\u001b[39m \u001b[43minsert_tool_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mres_next\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsgs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 246\u001b[0m res_next \u001b[38;5;241m=\u001b[39m chat_method(model\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgpt-4-0125-preview\u001b[39m\u001b[38;5;124m\"\u001b[39m, functions\u001b[38;5;241m=\u001b[39mfunctions, msgs\u001b[38;5;241m=\u001b[39mmsgs)\n\u001b[1;32m 247\u001b[0m \u001b[38;5;66;03m# for m in msgs:\u001b[39;00m\n\u001b[1;32m 248\u001b[0m \u001b[38;5;66;03m# print(m)\u001b[39;00m\n",
524+
"Cell \u001b[0;32mIn[3], line 77\u001b[0m, in \u001b[0;36minsert_tool_response\u001b[0;34m(res, msgs)\u001b[0m\n\u001b[1;32m 72\u001b[0m msgs\u001b[38;5;241m.\u001b[39mappend({\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtool\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtool_call_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(assistant_message\u001b[38;5;241m.\u001b[39mtool_calls[i]\u001b[38;5;241m.\u001b[39mid), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: assistant_message\u001b[38;5;241m.\u001b[39mtool_calls[i]\u001b[38;5;241m.\u001b[39mfunction\u001b[38;5;241m.\u001b[39mname, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOrder placed. the price is \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m(i\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;250m \u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m10\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m dollars.\u001b[39m\u001b[38;5;124m\"\u001b[39m})\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m tool_call\u001b[38;5;241m.\u001b[39mfunction\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maddition\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 74\u001b[0m msgs\u001b[38;5;241m.\u001b[39mappend({\n\u001b[1;32m 75\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtool\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 76\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maddition\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m---> 77\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[43madd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtool_call\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marguments\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 78\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtool_call_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: tool_call\u001b[38;5;241m.\u001b[39mid\n\u001b[1;32m 79\u001b[0m })\n\u001b[1;32m 80\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m tool_call\u001b[38;5;241m.\u001b[39mfunction\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msubtraction\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 81\u001b[0m msgs\u001b[38;5;241m.\u001b[39mappend({\n\u001b[1;32m 82\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtool\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 83\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msubtraction\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 84\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: sub(tool_call\u001b[38;5;241m.\u001b[39mfunction\u001b[38;5;241m.\u001b[39marguments),\n\u001b[1;32m 85\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtool_call_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: tool_call\u001b[38;5;241m.\u001b[39mid\n\u001b[1;32m 86\u001b[0m })\n",
525+
"Cell \u001b[0;32mIn[3], line 8\u001b[0m, in \u001b[0;36madd\u001b[0;34m(args)\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21madd\u001b[39m(args: \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m 7\u001b[0m args \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(args)\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mfloat\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43ma\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mfloat\u001b[39m(args[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m]))\n",
526+
"\u001b[0;31mValueError\u001b[0m: could not convert string to float: 'result'"
527+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
448528
]
449529
}
450530
],
451531
"source": [
532+
<<<<<<< HEAD
452533
<<<<<<< HEAD
453534
"user_query2 = \"now order 3 umbrellas for me and generate a password of length 8\"\n",
454535
"msgs = run_completion(get_mistral_rubra_response, user_query2, msgs)"
536+
=======
537+
"user_query3 = \"User tool to help me : What is four plus six? What is the result of that plus 2? Take the result and multiply by 5 and then divide by two\"\n",
538+
"msgs = run_completion(get_mistral_rubra_response, user_query3, msgs)"
539+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
455540
]
456541
},
457542
{
458543
"cell_type": "markdown",
459544
"metadata": {},
545+
<<<<<<< HEAD
460546
"source": [
461547
"## Simple Math Chaining"
462548
=======
@@ -612,6 +698,10 @@
612698
"print(functions)\n"
613699
]
614700
>>>>>>> ba3724d (a first working version integrated tree_sitter with python parser code)
701+
=======
702+
"outputs": [],
703+
"source": []
704+
>>>>>>> 7cb21bd (Hacky func streaming (#1))
615705
}
616706
],
617707
"metadata": {

0 commit comments

Comments
 (0)