Skip to content

refactor!: Removing deprecated InputRange, op_precision and input_shapes #607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 23, 2021
Merged

refactor!: Removing deprecated InputRange, op_precision and input_shapes #607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
621bc67
refactor!: Removing deprecated InputRange, op_precision and input_shapes
narendasan Aug 25, 2021
f8b1866
refactor(//tests/py): Update QAT Testscript
narendasan Sep 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 0 additions & 88 deletions cpp/include/trtorch/trtorch.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,64 +506,6 @@ struct TRTORCH_API CompileSpec {
bool explicit_set_dtype;
};

/**
* @brief A struct to hold an input range (used by TensorRT Optimization
* profile)
*
* This struct can either hold a single vector representing an input shape,
* signifying a static input shape or a set of three input shapes representing
* the min, optiminal and max input shapes allowed for the engine.
*/
struct TRTORCH_API InputRange {
/// Minimum acceptable input size into the engine
std::vector<int64_t> min;
/// Optimal input size into the engine (gets best performace)
std::vector<int64_t> opt;
/// Maximum acceptable input size into the engine
std::vector<int64_t> max;
/**
* @brief Construct a new Input Range object for static input size from
* vector
*
* @param opt
*/
[[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
std::vector<int64_t> opt);
/**
* @brief Construct a new Input Range object static input size from
* c10::ArrayRef (the type produced by tensor.sizes())
*
* @param opt
*/
[[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
c10::ArrayRef<int64_t> opt);
/**
* @brief Construct a new Input Range object dynamic input size from vectors
* for min, opt, and max supported sizes
*
* @param min
* @param opt
* @param max
*/
[[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
std::vector<int64_t> min,
std::vector<int64_t> opt,
std::vector<int64_t> max);
/**
* @brief Construct a new Input Range object dynamic input size from
* c10::ArrayRef (the type produced by tensor.sizes()) for min, opt, and max
* supported sizes
*
* @param min
* @param opt
* @param max
*/
[[deprecated("trtorch::CompileSpec::InputRange is being deprecated in favor of trtorch::CompileSpec::Input. trtorch::CompileSpec::InputRange will be removed in TRTorch v0.5.0")]] InputRange(
c10::ArrayRef<int64_t> min,
c10::ArrayRef<int64_t> opt,
c10::ArrayRef<int64_t> max);
};

/**
* @brief A struct to hold fallback info
*/
Expand Down Expand Up @@ -596,18 +538,6 @@ struct TRTORCH_API CompileSpec {
TorchFallback(bool enabled, uint64_t min_size) : enabled(enabled), min_block_size(min_size) {}
};

/**
* @brief Construct a new Extra Info object from input ranges.
* Each entry in the vector represents a input and should be provided in call
* order.
*
* Use this constructor if you want to use dynamic shape
*
* @param input_ranges
*/
[[deprecated("trtorch::CompileSpec::CompileSpec(std::vector<InputRange> input_ranges) is being deprecated in favor of trtorch::CompileSpec::CompileSpec(std::vector<Input> inputs). Please use CompileSpec(std::vector<Input> inputs). trtorch::CompileSpec::CompileSpec(std::vector<InputRange> input_ranges) will be removed in TRTorch v0.5.0")]] CompileSpec(
std::vector<InputRange> input_ranges)
: input_ranges(std::move(input_ranges)) {}
/**
* @brief Construct a new Extra Info object
* Convienence constructor to set fixed input size from vectors describing
Expand Down Expand Up @@ -657,24 +587,6 @@ struct TRTORCH_API CompileSpec {
*/
std::vector<Input> inputs;

/**
* Sizes for inputs to engine, can either be a single size or a range
* defined by Min, Optimal, Max sizes
*
* Order is should match call order
*/
[[deprecated(
"trtorch::CompileSpec::input_ranges is being deprecated in favor of trtorch::CompileSpec::inputs. trtorch::CompileSpec::input_ranges will be removed in TRTorch v0.5.0")]] std::
vector<InputRange>
input_ranges;

/**
* Default operating precision for the engine
*/
[[deprecated(
"trtorch::CompileSpec::op_precision is being deprecated in favor of trtorch::CompileSpec::enabled_precisions, a set of all enabled precisions to use during compilation, trtorch::CompileSpec::op_precision will be removed in TRTorch v0.5.0")]] DataType
op_precision = DataType::kFloat;

/**
* @brief The set of precisions TensorRT is allowed to use for kernels during compilation
*
Expand Down
55 changes: 2 additions & 53 deletions cpp/src/compile_spec.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,30 +144,6 @@ CompileSpec::Device::DeviceType::DeviceType(c10::DeviceType t) {
value = DeviceType::kGPU;
}

CompileSpec::InputRange::InputRange(std::vector<int64_t> opt) {
this->opt = opt;
this->min = opt;
this->max = opt;
}

CompileSpec::InputRange::InputRange(c10::IntArrayRef opt) {
this->opt = core::util::toVec(opt);
this->min = core::util::toVec(opt);
this->max = core::util::toVec(opt);
}

CompileSpec::InputRange::InputRange(std::vector<int64_t> min, std::vector<int64_t> opt, std::vector<int64_t> max) {
this->opt = opt;
this->min = min;
this->max = max;
}

CompileSpec::InputRange::InputRange(c10::IntArrayRef min, c10::IntArrayRef opt, c10::IntArrayRef max) {
this->opt = core::util::toVec(opt);
this->min = core::util::toVec(min);
this->max = core::util::toVec(max);
}

CompileSpec::CompileSpec(std::vector<c10::ArrayRef<int64_t>> fixed_sizes) {
for (auto in : fixed_sizes) {
inputs.push_back(Input(in));
Expand Down Expand Up @@ -289,22 +265,10 @@ CompileSpec::Input::Input(

/* ==========================================*/

core::ir::Input to_internal_input(CompileSpec::InputRange& i) {
return core::ir::Input(i.min, i.opt, i.max);
}

core::ir::Input to_internal_input(CompileSpec::Input& i) {
return core::ir::Input(i.min_shape, i.opt_shape, i.max_shape, toTRTDataType(i.dtype), toTRTTensorFormat(i.format));
}

std::vector<core::ir::Input> to_vec_internal_inputs(std::vector<CompileSpec::InputRange>& external) {
std::vector<core::ir::Input> internal;
for (auto range : external) {
internal.push_back(to_internal_input(range));
}
return internal;
}

std::vector<core::ir::Input> to_vec_internal_inputs(std::vector<CompileSpec::Input>& external) {
std::vector<core::ir::Input> internal;
for (auto range : external) {
Expand All @@ -328,24 +292,9 @@ core::runtime::CudaDevice to_internal_cuda_device(CompileSpec::Device device) {

core::CompileSpec to_internal_compile_spec(CompileSpec external) {
core::CompileSpec internal(to_vec_internal_inputs(external.inputs));
if (external.input_ranges.size() > 0 && external.inputs.size() > 0) {
TRTORCH_THROW_ERROR(
"Saw both input specs listed for inputs and input_ranges in CompileSpec. input_ranges is deprecated and will be removed in v0.5.0. Please port forward to using inputs");
} else if (external.input_ranges.size() > 0) {
internal = core::CompileSpec(to_vec_internal_inputs(external.input_ranges));
} else {
TRTORCH_CHECK(external.inputs.size() > 0, "Compilation requires at least one input specification");
internal = core::CompileSpec(to_vec_internal_inputs(external.inputs));
}

if (external.enabled_precisions.size() == 1 &&
toTRTDataType(*external.enabled_precisions.begin()) == nvinfer1::DataType::kFLOAT &&
toTRTDataType(external.op_precision) != nvinfer1::DataType::kFLOAT) {
internal.convert_info.engine_settings.enabled_precisions.insert(toTRTDataType(external.op_precision));
} else {
for (auto p : external.enabled_precisions) {
internal.convert_info.engine_settings.enabled_precisions.insert(toTRTDataType(p));
}
for (auto p : external.enabled_precisions) {
internal.convert_info.engine_settings.enabled_precisions.insert(toTRTDataType(p));
}

/* We want default behavior for types to match PyTorch, so in the case the user did not explicitly set the dtype for
Expand Down
2 changes: 1 addition & 1 deletion examples/int8/training/vgg16/test_qat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def test(model, dataloader, crit):
import trtorch
# trtorch.logging.set_reportable_log_level(trtorch.logging.Level.Debug)
compile_settings = {
"input_shapes": [[1, 3, 32, 32]],
"inputs": [trtorch.Input([1, 3, 32, 32])],
"op_precision": torch.int8 # Run with FP16
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can change this to enabled_precisions: {torch.int8}. I can do this in a separate PR if you want to merge this quick.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ill just add it to this one

}
new_mod = torch.jit.load('trained_vgg16_qat.jit.pt')
Expand Down
19 changes: 1 addition & 18 deletions py/trtorch/_compile_spec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,22 +157,11 @@ def _parse_torch_fallback(fallback_info: Dict[str, Any]) -> trtorch._C.TorchFall

def _parse_compile_spec(compile_spec: Dict[str, Any]) -> trtorch._C.CompileSpec:
info = trtorch._C.CompileSpec()
if "input_shapes" not in compile_spec and "inputs" not in compile_spec:
if "inputs" not in compile_spec:
raise KeyError(
"Module input definitions are requried to compile module. Provide a list of trtorch.Input keyed to \"inputs\" in the compile spec"
)

if "input_shapes" in compile_spec and "inputs" in compile_spec:
raise KeyError(
"Found both key \"input_shapes\", and \"inputs\" in compile spec, please port forward to using only \"inputs\""
)

if "input_shapes" in compile_spec:
warnings.warn(
"Key \"input_shapes\" is deprecated in favor of \"inputs\". Support for \"input_shapes\" will be removed in TRTorch v0.5.0",
DeprecationWarning)
info.inputs = _parse_input_ranges(compile_spec["input_shapes"])

if "inputs" in compile_spec:
info.inputs = [i._to_internal() for i in compile_spec["inputs"]]

Expand All @@ -181,12 +170,6 @@ def _parse_compile_spec(compile_spec: Dict[str, Any]) -> trtorch._C.CompileSpec:
"Found both key \"op_precision\", and \"enabled_precisions\" in compile spec, please port forward to using only \"enabled_precisions\""
)

if "op_precision" in compile_spec:
warnings.warn(
"Key \"op_precision\" is being deprecated in favor of \"enabled_precision\" which expects a set of precisions to be enabled during compilation (FP32 will always be enabled), Support for \"op_precision\" will be removed in TRTorch v0.5.0",
DeprecationWarning)
info.enabled_precisions = _parse_enabled_precisions(compile_spec["op_precision"])

if "enabled_precisions" in compile_spec:
info.enabled_precisions = _parse_enabled_precisions(compile_spec["enabled_precisions"])
# We want default behavior to match PyTorch, so in the case the user did not explicitly set the dtype for inputs they
Expand Down
32 changes: 0 additions & 32 deletions tests/py/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,38 +13,6 @@ def setUp(self):
self.traced_model = torch.jit.trace(self.model, [self.input])
self.scripted_model = torch.jit.script(self.model)

def test_compile_traced_deprecated(self):
compile_spec = {
"input_shapes": [self.input.shape],
"device": {
"device_type": trtorch.DeviceType.GPU,
"gpu_id": 0,
"dla_core": 0,
"allow_gpu_fallback": False,
"disable_tf32": False
}
}

trt_mod = trtorch.compile(self.traced_model, compile_spec)
same = (trt_mod(self.input) - self.traced_model(self.input)).abs().max()
self.assertTrue(same < 2e-3)

def test_compile_script_deprecated(self):
compile_spec = {
"input_shapes": [self.input.shape],
"device": {
"device_type": trtorch.DeviceType.GPU,
"gpu_id": 0,
"dla_core": 0,
"allow_gpu_fallback": False,
"disable_tf32": False
}
}

trt_mod = trtorch.compile(self.scripted_model, compile_spec)
same = (trt_mod(self.input) - self.scripted_model(self.input)).abs().max()
self.assertTrue(same < 2e-3)

def test_compile_traced(self):
compile_spec = {
"inputs": [trtorch.Input(self.input.shape, dtype=torch.float, format=torch.contiguous_format)],
Expand Down
2 changes: 1 addition & 1 deletion tests/py/test_qat_trt_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def test_compile_script(self):

compile_spec = {
"inputs": [trtorch.Input([16, 3, 32, 32])],
"op_precision": torch.int8,
"enabled_precisions": {torch.int8},
# "enabled_precision": {torch.float32, torch.int8},
}

Expand Down