@@ -473,6 +473,14 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
473473 else { invalid_param = true ; }
474474 return true ;
475475 }
476+ if (arg == " --attention"
477+ CHECK_ARG
478+ std::string value (argv[i]);
479+ /* */ if (value == " causal" attention_type = LLAMA_ATTENTION_TYPE_CAUSAL; }
480+ else if (value == " non-causal" attention_type = LLAMA_ATTENTION_TYPE_NON_CAUSAL; }
481+ else { invalid_param = true ; }
482+ return true ;
483+ }
476484 if (arg == " --defrag-thold" " -dt"
477485 CHECK_ARG
478486 params.defrag_thold = std::stof (argv[i]);
@@ -758,7 +766,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
758766 params.cache_type_v = argv[++i];
759767 return true ;
760768 }
761- if (arg == " --multiline-input"
769+ if (arg == " -mli " || arg == " - -multiline-input"
762770 params.multiline_input = true ;
763771 return true ;
764772 }
@@ -1395,7 +1403,9 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
13951403 options.push_back ({ " *" " --keep N" " number of tokens to keep from the initial prompt (default: %d, -1 = all)" n_keep });
13961404 options.push_back ({ " *" " --chunks N" " max number of chunks to process (default: %d, -1 = all)" n_chunks });
13971405 options.push_back ({ " *" " -fa, --flash-attn" " enable Flash Attention (default: %s)" flash_attn ? " enabled" " disabled"
1398- options.push_back ({ " *" " -p, --prompt PROMPT" " prompt to start generation with (default: '%s')" prompt .c_str () });
1406+ options.push_back ({ " *" " -p, --prompt PROMPT" " prompt to start generation with\n "
1407+ " in conversation mode, this will be used as system prompt\n "
1408+ " (default: '%s')" prompt .c_str () });
13991409 options.push_back ({ " *" " -f, --file FNAME" " a file containing the prompt (default: none)"
14001410 options.push_back ({ " *" " --in-file FNAME" " an input file (repeat to specify multiple files)"
14011411 options.push_back ({ " *" " -bf, --binary-file FNAME" " binary file containing the prompt (default: none)"
@@ -1410,7 +1420,9 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
14101420 " halt generation at PROMPT, return control in interactive mode\n "
14111421 " can be specified more than once for multiple prompts"
14121422 options.push_back ({ " main" " -sp, --special" " special tokens output enabled (default: %s)" special ? " true" " false"
1413- options.push_back ({ " main" " -cnv, --conversation" " run in conversation mode (does not print special tokens and suffix/prefix, use default chat template) (default: %s)" conversation ? " true" " false"
1423+ options.push_back ({ " main" " -cnv, --conversation" " run in conversation mode, does not print special tokens and suffix/prefix\n "
1424+ " if suffix/prefix are not specified, default chat template will be used\n "
1425+ " (default: %s)" conversation ? " true" " false"
14141426 options.push_back ({ " main infill" " -i, --interactive" " run in interactive mode (default: %s)" interactive ? " true" " false"
14151427 options.push_back ({ " main infill" " -if, --interactive-first" " run in interactive mode and wait for input right away (default: %s)" interactive_first ? " true" " false"
14161428 options.push_back ({ " main infill" " -mli, --multiline-input" " allows you to write or paste multiple lines without ending each in '\\ '"
@@ -1454,6 +1466,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
14541466 options.push_back ({ " main" " --cfg-scale N" " strength of guidance (default: %.1f, 1.0 = disable)" double )sparams.cfg_scale });
14551467 options.push_back ({ " main" " --chat-template JINJA_TEMPLATE"
14561468 " set custom jinja chat template (default: template taken from model's metadata)\n "
1469+ " if suffix/prefix are specified, template will be disabled\n "
14571470 " only commonly used templates are accepted:\n "
14581471 " https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template"
14591472 options.push_back ({ " grammar"
@@ -1464,8 +1477,10 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
14641477 " For schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead"
14651478
14661479 options.push_back ({ " embedding"
1467- options.push_back ({ " embedding" " --pooling {none,mean,cls}"
1480+ options.push_back ({ " embedding" " --pooling {none,mean,cls,last }"
14681481 " pooling type for embeddings, use model default if unspecified"
1482+ options.push_back ({ " embedding" " --attention {causal,non-causal}"
1483+ " attention type for embeddings, use model default if unspecified"
14691484
14701485 options.push_back ({ " context hacking"
14711486 options.push_back ({ " *" " --rope-scaling {none,linear,yarn}"
@@ -2071,7 +2086,24 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_par
20712086 if (params.warmup ) {
20722087 LOG (" warming up the model with an empty run\n "
20732088
2074- std::vector<llama_token> tmp = { llama_token_bos (model), llama_token_eos (model), };
2089+ std::vector<llama_token> tmp;
2090+ llama_token bos = llama_token_bos (model);
2091+ llama_token eos = llama_token_eos (model);
2092+ // some models (e.g. T5) don't have a BOS token
2093+ if (bos != -1 ) {
2094+ tmp.push_back (bos);
2095+ }
2096+ tmp.push_back (eos);
2097+
2098+ if (llama_model_has_encoder (model)) {
2099+ llama_encode (lctx, llama_batch_get_one (tmp.data (), tmp.size (), 0 , 0 ));
2100+ llama_token decoder_start_token_id = llama_model_decoder_start_token (model);
2101+ if (decoder_start_token_id == -1 ) {
2102+ decoder_start_token_id = bos;
2103+ }
2104+ tmp.clear ();
2105+ tmp.push_back (decoder_start_token_id);
2106+ }
20752107 llama_decode (lctx, llama_batch_get_one (tmp.data (), std::min (tmp.size (), (size_t ) params.n_batch ), 0 , 0 ));
20762108 llama_kv_cache_clear (lctx);
20772109 llama_synchronize (lctx);
@@ -2154,6 +2186,7 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
21542186 cparams.yarn_beta_slow = params.yarn_beta_slow ;
21552187 cparams.yarn_orig_ctx = params.yarn_orig_ctx ;
21562188 cparams.pooling_type = params.pooling_type ;
2189+ cparams.attention_type = params.attention_type ;
21572190 cparams.defrag_thold = params.defrag_thold ;
21582191 cparams.cb_eval = params.cb_eval ;
21592192 cparams.cb_eval_user_data = params.cb_eval_user_data ;
0 commit comments