Skip to content

Add support for response sender in the default mode #364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jun 6, 2024
Merged

Add support for response sender in the default mode #364

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
01ba273
Add response sender to non-decoupled models and unify data pipelines …
kthui May 31, 2024
4551e04
Fix decoupled batch statistics to account for implicit batch size (#361)
kthui May 31, 2024
9f2865d
Fix decoupled gpu output error handling (#362)
kthui Jun 3, 2024
4961e24
Response sender to check for improper non-decoupled model usage (#363)
kthui Jun 4, 2024
61f46a7
Update copyright
kthui Jun 4, 2024
f5e6e3b
Rename decoupled monitor to queue monitor
kthui Jun 5, 2024
4d73203
Add documentation
kthui Jun 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 7 additions & 12 deletions src/infer_request.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ InferRequest::InferRequest(
pb_cancel_ =
std::make_shared<PbCancel>(response_factory_address_, request_address_);
response_sender_ = std::make_shared<ResponseSender>(
request_address_, response_factory_address_,
request_address_, response_factory_address_, nullptr /* is_decoupled */,
Stub::GetOrCreateInstance()->SharedMemory(), pb_cancel_);
#endif
}
Expand Down Expand Up @@ -272,7 +272,8 @@ InferRequest::SaveToSharedMemory(std::unique_ptr<SharedMemoryManager>& shm_pool)
std::unique_ptr<InferRequest>
InferRequest::LoadFromSharedMemory(
std::unique_ptr<SharedMemoryManager>& shm_pool,
bi::managed_external_buffer::handle_t request_handle, bool open_cuda_handle)
bi::managed_external_buffer::handle_t request_handle, bool open_cuda_handle,
bool const* is_model_decoupled)
{
AllocatedSharedMemory<char> infer_request_shm =
shm_pool->Load<char>(request_handle);
Expand Down Expand Up @@ -328,7 +329,7 @@ InferRequest::LoadFromSharedMemory(
return std::unique_ptr<InferRequest>(new InferRequest(
infer_request_shm, request_id_shm, correlation_id_shm,
requested_output_names_shm, model_name_shm, input_tensors, parameters_shm,
infer_trace_shm));
infer_trace_shm, is_model_decoupled));
}

InferRequest::InferRequest(
Expand All @@ -339,7 +340,8 @@ InferRequest::InferRequest(
std::unique_ptr<PbString>& model_name_shm,
std::vector<std::shared_ptr<PbTensor>>& input_tensors,
std::unique_ptr<PbString>& parameters_shm,
std::unique_ptr<InferenceTrace>& infer_trace_shm)
std::unique_ptr<InferenceTrace>& infer_trace_shm,
bool const* is_model_decoupled)
: infer_request_shm_(std::move(infer_request_shm)),
request_id_shm_(std::move(request_id_shm)),
requested_output_names_shm_(std::move(requested_output_names_shm)),
Expand Down Expand Up @@ -387,7 +389,7 @@ InferRequest::InferRequest(
pb_cancel_ =
std::make_shared<PbCancel>(response_factory_address_, request_address_);
response_sender_ = std::make_shared<ResponseSender>(
request_address_, response_factory_address_,
request_address_, response_factory_address_, is_model_decoupled,
Stub::GetOrCreateInstance()->SharedMemory(), pb_cancel_);
#endif
}
Expand All @@ -402,13 +404,6 @@ InferRequest::IsCancelled()
std::shared_ptr<ResponseSender>
InferRequest::GetResponseSender()
{
std::unique_ptr<Stub>& stub = Stub::GetOrCreateInstance();
if (!stub->IsDecoupled()) {
throw PythonBackendException(
"'get_response_sender' function must be called only when the model is "
"using the decoupled transaction policy.");
}

return response_sender_;
}

Expand Down
5 changes: 3 additions & 2 deletions src/infer_request.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class InferRequest {
static std::unique_ptr<InferRequest> LoadFromSharedMemory(
std::unique_ptr<SharedMemoryManager>& shm_pool,
bi::managed_external_buffer::handle_t request_handle,
bool open_cuda_handle);
bool open_cuda_handle, bool const* is_model_decoupled);

/// Disallow copying the inference request object.
DISALLOW_COPY_AND_ASSIGN(InferRequest);
Expand All @@ -135,7 +135,8 @@ class InferRequest {
std::unique_ptr<PbString>& model_name_shm,
std::vector<std::shared_ptr<PbTensor>>& input_tensors,
std::unique_ptr<PbString>& parameters_shm,
std::unique_ptr<InferenceTrace>& infer_trace_shm);
std::unique_ptr<InferenceTrace>& infer_trace_shm,
bool const* is_model_decoupled);

std::string request_id_;
CorrelationId correlation_id_;
Expand Down
Loading
Loading