Skip to content

Commit ff46cba

Browse files
authored
[jinja] Add is mapping test (#995)
Closes #993
1 parent dd1befa commit ff46cba

File tree

3 files changed

+135
-61
lines changed

3 files changed

+135
-61
lines changed

packages/jinja/src/runtime.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ export class Environment {
274274
["string", (operand) => operand.type === "StringValue"],
275275
["number", (operand) => operand.type === "NumericValue"],
276276
["integer", (operand) => operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value)],
277-
["iterable", (operand) => operand instanceof ArrayValue || operand instanceof StringValue],
277+
["iterable", (operand) => operand.type === "ArrayValue" || operand.type === "StringValue"],
278+
["mapping", (operand) => operand.type === "ObjectValue"],
278279
[
279280
"lower",
280281
(operand) => {

packages/jinja/test/e2e.test.js

Lines changed: 98 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -76,15 +76,8 @@ const EXAMPLE_FUNCTION_CALLING_WITH_SYSTEM = [
7676
{ role: "user", content: "Hi, can you tell me the current stock price of AAPL?" },
7777
];
7878

79-
// Adapted from https://huggingface.co/CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF
80-
const EXAMPLE_CHAT_WITH_TOOLS = [
81-
{
82-
role: "user",
83-
content: "What's the weather like in Oslo and Stockholm?",
84-
},
85-
];
86-
const EXAMPLE_TOOLS = [
87-
{
79+
const EXAMPLE_TOOL_JSON_SCHEMAS = {
80+
get_current_weather: {
8881
type: "function",
8982
function: {
9083
name: "get_current_weather",
@@ -105,7 +98,81 @@ const EXAMPLE_TOOLS = [
10598
},
10699
},
107100
},
101+
get_current_temperature_v1: {
102+
type: "function",
103+
function: {
104+
name: "get_current_temperature",
105+
description: "Get the current temperature at a location.",
106+
parameters: {
107+
type: "object",
108+
properties: {
109+
location: {
110+
type: "string",
111+
description: 'The location to get the temperature for, in the format "City, Country"',
112+
},
113+
},
114+
required: ["location"],
115+
},
116+
return: {
117+
type: "number",
118+
description: "The current temperature at the specified location in the specified units, as a float.",
119+
},
120+
},
121+
},
122+
get_current_temperature_v2: {
123+
type: "function",
124+
function: {
125+
name: "get_current_temperature",
126+
description: "Get the current temperature at a location.",
127+
parameters: {
128+
type: "object",
129+
properties: {
130+
location: {
131+
type: "string",
132+
description: 'The location to get the temperature for, in the format "City, Country"',
133+
},
134+
unit: {
135+
type: "string",
136+
enum: ["celsius", "fahrenheit"],
137+
description: "The unit to return the temperature in.",
138+
},
139+
},
140+
required: ["location", "unit"],
141+
},
142+
return: {
143+
type: "number",
144+
description: "The current temperature at the specified location in the specified units, as a float.",
145+
},
146+
},
147+
},
148+
get_current_wind_speed: {
149+
type: "function",
150+
function: {
151+
name: "get_current_wind_speed",
152+
description: "Get the current wind speed in km/h at a given location.",
153+
parameters: {
154+
type: "object",
155+
properties: {
156+
location: {
157+
type: "string",
158+
description: 'The location to get the temperature for, in the format "City, Country"',
159+
},
160+
},
161+
required: ["location"],
162+
},
163+
return: {
164+
type: "number",
165+
description: "The current wind speed at the given location in km/h, as a float.",
166+
},
167+
},
168+
},
169+
};
170+
171+
const EXAMPLE_LIST_OF_TOOLS = [
172+
EXAMPLE_TOOL_JSON_SCHEMAS.get_current_temperature_v2,
173+
EXAMPLE_TOOL_JSON_SCHEMAS.get_current_wind_speed,
108174
];
175+
109176
/**
110177
* Defined in https://github.com/huggingface/transformers
111178
* Keys correspond to `model_type` in the transformers repo.
@@ -532,55 +599,7 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({
532599
},
533600
{ role: "tool", tool_call_id: "abcdef123", name: "get_current_temperature", content: "22.0" },
534601
],
535-
tools: [
536-
{
537-
type: "function",
538-
function: {
539-
name: "get_current_temperature",
540-
description: "Get the current temperature at a location.",
541-
parameters: {
542-
type: "object",
543-
properties: {
544-
location: {
545-
type: "string",
546-
description: 'The location to get the temperature for, in the format "City, Country"',
547-
},
548-
unit: {
549-
type: "string",
550-
enum: ["celsius", "fahrenheit"],
551-
description: "The unit to return the temperature in.",
552-
},
553-
},
554-
required: ["location", "unit"],
555-
},
556-
return: {
557-
type: "number",
558-
description: "The current temperature at the specified location in the specified units, as a float.",
559-
},
560-
},
561-
},
562-
{
563-
type: "function",
564-
function: {
565-
name: "get_current_wind_speed",
566-
description: "Get the current wind speed in km/h at a given location.",
567-
parameters: {
568-
type: "object",
569-
properties: {
570-
location: {
571-
type: "string",
572-
description: 'The location to get the temperature for, in the format "City, Country"',
573-
},
574-
},
575-
required: ["location"],
576-
},
577-
return: {
578-
type: "number",
579-
description: "The current wind speed at the given location in km/h, as a float.",
580-
},
581-
},
582-
},
583-
],
602+
tools: EXAMPLE_LIST_OF_TOOLS,
584603
bos_token: "<s>",
585604
eos_token: "</s>",
586605
},
@@ -590,8 +609,13 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({
590609
"CISCai/Mistral-7B-Instruct-v0.3-SOTA-GGUF": {
591610
chat_template: `{{ bos_token }}{% set ns = namespace(lastuser=-1, system=false, functions=false) %}{% if tools %}{% for message in messages %}{% if message['role'] == 'user' %}{% set ns.lastuser = loop.index0 %}{% elif message['role'] == 'system' %}{% set ns.system = message['content'] %}{% endif %}{% endfor %}{% set ns.functions = tools|selectattr('type','eq','function')|map(attribute='function')|list|tojson %}{% endif %}{% for message in messages %}{% if message['role'] == 'user' %}{% if loop.index0 == ns.lastuser and ns.functions %}{{ '[AVAILABLE_TOOLS] ' }}{{ ns.functions }}{{ '[/AVAILABLE_TOOLS]' }}{% endif %}{{ '[INST] ' }}{% if loop.index0 == ns.lastuser and ns.system %}{{ ns.system + ' ' }}{% endif %}{{ message['content'] }}{{ '[/INST]' }}{% elif message['role'] == 'tool' %}{{ '[TOOL_RESULTS] ' }}{{ dict(call_id=message['tool_call_id'], content=message['content'])|tojson }}{{ '[/TOOL_RESULTS]' }}{% elif message['role'] == 'assistant' %}{% if message['tool_calls'] %}{{ '[TOOL_CALLS] [' }}{% for call in message['tool_calls'] %}{% if call['type'] == 'function' %}{{ dict(id=call['id'], name=call['function']['name'], arguments=call['function']['arguments'])|tojson }}{% endif %}{% if not loop.last %}{{ ', ' }}{% endif %}{% endfor %}{{ ']' }}{% else %}{{ message['content'] }}{% endif %}{{ eos_token }}{% endif %}{% endfor %}`,
592611
data: {
593-
messages: EXAMPLE_CHAT_WITH_TOOLS,
594-
tools: EXAMPLE_TOOLS,
612+
messages: [
613+
{
614+
role: "user",
615+
content: "What's the weather like in Oslo and Stockholm?",
616+
},
617+
],
618+
tools: [EXAMPLE_TOOL_JSON_SCHEMAS.get_current_weather],
595619
bos_token: "<s>",
596620
eos_token: "</s>",
597621
},
@@ -630,6 +654,20 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({
630654
},
631655
target: `<|begin_of_text|>You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> {"type": "function", "function": {"name": get_stock_fundamentals", "description": "get_stock_fundamentals(symbol: str) -> dict - Get fundamental data for a given stock symbol using yfinance API.\n\n Args:\n symbol(str): The stock symbol.\n Returns:\n A dictionary containing fundamental data.\n\nKeys:\n - 'symbol': The stock symbol.\n - 'company_name': The long name of the company.\n - 'sector': The sector to which the company belongs.\n - 'industry': The industry to which the company belongs.\n - 'market_cap': The market capitalization of the company.\n - 'pe_ratio': The forward price-to-earnings ratio.\n - 'pb_ratio': The price-to-book ratio.\n - 'dividend_yield': The dividend yield.\n - 'eps': The trailing earnings per share.\n - 'beta': The beta value of the stock.\n - '52_week_high': The 52-week high price of the stock.\n - '52_week_low': The 52-week low price of the stock.", "parameters": {"type": "object", "properties": {"symbol": {"type": "string", "description": "The stock symbol."}}, "required": ["symbol"]}} </tools>Use the following pydantic model json schema for each tool call you will make: {"properties": {"arguments": {"title": "Arguments", "type": "object"}, "name": {"title": "Name", "type": "string"}}, "required": ["arguments", "name"], "title": "FunctionCall", "type": "object"}\nFor each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:\n<tool_call>\n{"arguments": <args-dict>, "name": <function-name>}\n</tool_call><|im_end|><|im_start|>user\nFetch the stock fundamentals data for Tesla (TSLA)<|im_end|>\n<|im_start|>assistant\n`,
632656
},
657+
"meta-llama/Llama-3.1-8B-Instruct": {
658+
chat_template: `{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = "26 Jul 2024" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = "" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- "<|start_header_id|>system<|end_header_id|>\\n\\n" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- "Environment: ipython\\n" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\\n\\n"}}\n{%- endif %}\n{{- "Cutting Knowledge Date: December 2023\\n" }}\n{{- "Today Date: " + date_string + "\\n\\n" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}\n {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}\n {{- "Do not use variables.\\n\\n" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- "\\n\\n" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- "<|eot_id|>" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- "Given the following functions, please respond with a JSON for a function call " }}\n {{- "with its proper arguments that best answers the given prompt.\\n\\n" }}\n {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}\n {{- "Do not use variables.\\n\\n" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- "\\n\\n" }}\n {%- endfor %}\n {{- first_user_message + "<|eot_id|>"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception("This model only supports single tool-calls at once!") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- "<|python_tag|>" + tool_call.name + ".call(" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '="' + arg_val + '"' }}\n {%- if not loop.last %}\n {{- ", " }}\n {%- endif %}\n {%- endfor %}\n {{- ")" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{"name": "' + tool_call.name + '", ' }}\n {{- '"parameters": ' }}\n {{- tool_call.arguments | tojson }}\n {{- "}" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- "<|eom_id|>" }}\n {%- else %}\n {{- "<|eot_id|>" }}\n {%- endif %}\n {%- elif message.role == "tool" or message.role == "ipython" %}\n {{- "<|start_header_id|>ipython<|end_header_id|>\\n\\n" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- "<|eot_id|>" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n`,
659+
data: {
660+
messages: [
661+
{ role: "system", content: "You are a bot that responds to weather queries." },
662+
{ role: "user", content: "Hey, what's the temperature in Paris right now?" },
663+
],
664+
tools: [EXAMPLE_TOOL_JSON_SCHEMAS.get_current_temperature_v1],
665+
bos_token: "<|begin_of_text|>",
666+
eos_token: "<|im_end|>",
667+
add_generation_prompt: true,
668+
},
669+
target: `<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are a bot that responds to weather queries.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {"name": function name, "parameters": dictionary of argument name and its value}.Do not use variables.\n\n{\n "type": "function",\n "function": {\n "name": "get_current_temperature",\n "description": "Get the current temperature at a location.",\n "parameters": {\n "type": "object",\n "properties": {\n "location": {\n "type": "string",\n "description": "The location to get the temperature for, in the format \\"City, Country\\""\n }\n },\n "required": [\n "location"\n ]\n },\n "return": {\n "type": "number",\n "description": "The current temperature at the specified location in the specified units, as a float."\n }\n }\n}\n\nHey, what's the temperature in Paris right now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`,
670+
},
633671
});
634672

635673
describe("End-to-end tests", () => {

packages/jinja/test/templates.test.js

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ const TEST_STRINGS = {
101101
IS_OPERATOR_3: `|{{ 1 is odd }}|{{ 2 is odd }}|{{ 1 is even }}|{{ 2 is even }}|{{ 2 is number }}|{{ '2' is number }}|{{ 2 is integer }}|{{ '2' is integer }}|`,
102102
IS_OPERATOR_4: `|{{ func is callable }}|{{ 2 is callable }}|{{ 1 is iterable }}|{{ 'hello' is iterable }}|`,
103103
IS_OPERATOR_5: `|{{ 'a' is lower }}|{{ 'A' is lower }}|{{ 'a' is upper }}|{{ 'A' is upper }}|`,
104+
IS_OPERATOR_6: `|{{ string is mapping }}|{{ number is mapping }}|{{ array is mapping }}|{{ dict is mapping }}|`,
104105

105106
// Short-circuit evaluation
106107
SHORT_CIRCUIT: `{{ false and raise_exception('This should not be printed') }}`,
@@ -2073,6 +2074,33 @@ const TEST_PARSED = {
20732074
{ value: "}}", type: "CloseExpression" },
20742075
{ value: "|", type: "Text" },
20752076
],
2077+
IS_OPERATOR_6: [
2078+
{ value: "|", type: "Text" },
2079+
{ value: "{{", type: "OpenExpression" },
2080+
{ value: "string", type: "Identifier" },
2081+
{ value: "is", type: "Is" },
2082+
{ value: "mapping", type: "Identifier" },
2083+
{ value: "}}", type: "CloseExpression" },
2084+
{ value: "|", type: "Text" },
2085+
{ value: "{{", type: "OpenExpression" },
2086+
{ value: "number", type: "Identifier" },
2087+
{ value: "is", type: "Is" },
2088+
{ value: "mapping", type: "Identifier" },
2089+
{ value: "}}", type: "CloseExpression" },
2090+
{ value: "|", type: "Text" },
2091+
{ value: "{{", type: "OpenExpression" },
2092+
{ value: "array", type: "Identifier" },
2093+
{ value: "is", type: "Is" },
2094+
{ value: "mapping", type: "Identifier" },
2095+
{ value: "}}", type: "CloseExpression" },
2096+
{ value: "|", type: "Text" },
2097+
{ value: "{{", type: "OpenExpression" },
2098+
{ value: "dict", type: "Identifier" },
2099+
{ value: "is", type: "Is" },
2100+
{ value: "mapping", type: "Identifier" },
2101+
{ value: "}}", type: "CloseExpression" },
2102+
{ value: "|", type: "Text" },
2103+
],
20762104

20772105
// Short-circuit evaluation
20782106
SHORT_CIRCUIT: [
@@ -2927,6 +2955,12 @@ const TEST_CONTEXT = {
29272955
func: () => {},
29282956
},
29292957
IS_OPERATOR_5: {},
2958+
IS_OPERATOR_6: {
2959+
string: "hello",
2960+
number: 1,
2961+
array: [1, 2, 3],
2962+
dict: { a: 1 },
2963+
},
29302964

29312965
// Short-circuit evaluation
29322966
SHORT_CIRCUIT: {},
@@ -3087,6 +3121,7 @@ const EXPECTED_OUTPUTS = {
30873121
IS_OPERATOR_3: `|true|false|false|true|true|false|true|false|`,
30883122
IS_OPERATOR_4: `|true|false|false|true|`,
30893123
IS_OPERATOR_5: `|true|false|false|true|`,
3124+
IS_OPERATOR_6: `|false|false|false|true|`,
30903125

30913126
// Short-circuit evaluation
30923127
SHORT_CIRCUIT: `false`,

0 commit comments

Comments
 (0)