Skip to content

Commit 6152756

Browse files
rgerganovrgerganov
authored
Merge pull request #1 from ggerganov/sl/rpc-backend-cpy
fix copy_tensor being called on the src buffer instead of the dst buffer
2 parents 6c276de + 5f8720f commit 6152756

File tree

5 files changed

+36
-19
lines changed

5 files changed

+36
-19
lines changed

‎Makefile

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ ifeq ($(UNAME_S),Darwin)
6767
endif
6868
endif
6969

70+
ifdef LLAMA_RPC
71+
BUILD_TARGETS += rpc-server
72+
endif
73+
7074
default: $(BUILD_TARGETS)
7175

7276
test: $(TEST_TARGETS)
@@ -416,6 +420,11 @@ ifdef LLAMA_BLIS
416420
MK_LDFLAGS += -lblis -L/usr/local/lib
417421
endif # LLAMA_BLIS
418422

423+
ifdef LLAMA_RPC
424+
MK_CPPFLAGS += -DGGML_USE_RPC
425+
OBJS += ggml-rpc.o
426+
endif # LLAMA_RPC
427+
419428
ifdef LLAMA_CUBLAS
420429
# LLAMA_CUBLAS is deprecated and will be removed in the future
421430
LLAMA_CUDA := 1
@@ -626,11 +635,26 @@ ggml-metal-embed.o: ggml-metal.metal ggml-common.h
626635
endif
627636
endif # LLAMA_METAL
628637

638+
OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o unicode.o unicode-data.o
639+
COMMON_H_DEPS = common/common.h common/sampling.h common/log.h llama.h
640+
COMMON_DEPS = common.o sampling.o grammar-parser.o build-info.o json-schema-to-grammar.o
641+
629642
ifndef LLAMA_NO_LLAMAFILE
630643
sgemm.o: sgemm.cpp sgemm.h ggml.h
631644
$(CXX) $(CXXFLAGS) -c $< -o $@
632645
endif
633646

647+
ifdef LLAMA_RPC
648+
ggml-rpc.o: ggml-rpc.cpp ggml-rpc.h
649+
$(CXX) $(CXXFLAGS) -c $< -o $@
650+
651+
rpc-server.o: examples/rpc/rpc-server.cpp ggml-rpc.h
652+
$(CXX) $(CXXFLAGS) -c $< -o $@
653+
654+
rpc-server: rpc-server.o ggml.o llama.o $(COMMON_DEPS) $(OBJS)
655+
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
656+
endif # LLAMA_RPC
657+
634658
GF_CC := $(CC)
635659
include scripts/get-flags.mk
636660

@@ -710,14 +734,9 @@ unicode.o: unicode.cpp unicode.h
710734
unicode-data.o: unicode-data.cpp unicode-data.h
711735
$(CXX) $(CXXFLAGS) -c $< -o $@
712736

713-
OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o unicode.o unicode-data.o
714-
715737
llama.o: llama.cpp unicode.h ggml.h ggml-alloc.h ggml-backend.h ggml-cuda.h ggml-metal.h llama.h
716738
$(CXX) $(CXXFLAGS) -c $< -o $@
717739

718-
COMMON_H_DEPS = common/common.h common/sampling.h common/log.h llama.h
719-
COMMON_DEPS = common.o sampling.o grammar-parser.o build-info.o json-schema-to-grammar.o
720-
721740
common.o: common/common.cpp $(COMMON_H_DEPS)
722741
$(CXX) $(CXXFLAGS) -c $< -o $@
723742

‎ggml-alloc.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -750,7 +750,7 @@ static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor *
750750
// this tensor was allocated without ggml-backend
751751
return;
752752
}
753-
ggml_backend_view_init(galloc->buffers[buffer_id], tensor);
753+
ggml_backend_view_init(tensor);
754754
}
755755
} else {
756756
if (tensor->data == NULL) {
@@ -899,12 +899,12 @@ static bool alloc_tensor_range(struct ggml_context * ctx,
899899
if (t->view_src == NULL) {
900900
ggml_tallocr_alloc(&tallocr, t);
901901
} else if (t->buffer == NULL) {
902-
ggml_backend_view_init(buffer, t);
902+
ggml_backend_view_init(t);
903903
}
904904
} else {
905905
if (t->view_src != NULL && t->buffer == NULL) {
906906
// view of a pre-allocated tensor
907-
ggml_backend_view_init(buffer, t);
907+
ggml_backend_view_init(t);
908908
}
909909
}
910910
}

‎ggml-backend.c

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {
151151
bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {
152152
ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;
153153
if (dst_buf->iface.cpy_tensor) {
154-
return src->buffer->iface.cpy_tensor(dst_buf, src, dst);
154+
return dst_buf->iface.cpy_tensor(dst_buf, src, dst);
155155
}
156156
return false;
157157
}
@@ -321,9 +321,7 @@ void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst
321321
ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));
322322
} else if (ggml_backend_buffer_is_host(dst->buffer)) {
323323
ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));
324-
}
325-
bool same_backend = strcmp(ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer)) == 0;
326-
if (!same_backend || !ggml_backend_buffer_copy_tensor(src, dst)) {
324+
} else if (!ggml_backend_buffer_copy_tensor(src, dst)) {
327325
#ifndef NDEBUG
328326
fprintf(stderr, "%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));
329327
#endif
@@ -1889,15 +1887,15 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched,
18891887

18901888
// utils
18911889

1892-
void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
1890+
void ggml_backend_view_init(struct ggml_tensor * tensor) {
18931891
GGML_ASSERT(tensor->buffer == NULL);
18941892
GGML_ASSERT(tensor->view_src != NULL);
18951893
GGML_ASSERT(tensor->view_src->buffer != NULL);
18961894
GGML_ASSERT(tensor->view_src->data != NULL);
18971895

1898-
tensor->buffer = buffer;
1896+
tensor->buffer = tensor->view_src->buffer;
18991897
tensor->data = (char *)tensor->view_src->data + tensor->view_offs;
1900-
ggml_backend_buffer_init_tensor(buffer, tensor);
1898+
ggml_backend_buffer_init_tensor(tensor->buffer, tensor);
19011899
}
19021900

19031901
void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {
@@ -1956,7 +1954,7 @@ static void graph_copy_init_tensor(struct ggml_hash_set hash_set, struct ggml_te
19561954
struct ggml_tensor * dst = node_copies[id];
19571955
if (dst->view_src != NULL) {
19581956
graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);
1959-
ggml_backend_view_init(dst->view_src->buffer, dst);
1957+
ggml_backend_view_init(dst);
19601958
}
19611959
else {
19621960
ggml_backend_tensor_copy(src, dst);

‎ggml-backend.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ extern "C" {
225225

226226
// Tensor initialization
227227
GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
228-
GGML_API void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
228+
GGML_API void ggml_backend_view_init(struct ggml_tensor * tensor);
229229

230230

231231
#ifdef __cplusplus

‎ggml-rpc.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer
491491
if (remote_ptr != 0) {
492492
ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft,
493493
ggml_backend_rpc_buffer_interface,
494-
new ggml_backend_rpc_buffer_context{sock, {}, remote_ptr, "RPC"},
494+
new ggml_backend_rpc_buffer_context{sock, {}, remote_ptr, "RPC[" + std::string(buft_ctx->endpoint) + "]"},
495495
remote_size);
496496
return buffer;
497497
} else {
@@ -692,7 +692,7 @@ GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const
692692
GGML_CALL ggml_backend_t ggml_backend_rpc_init(const char * endpoint) {
693693
ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context {
694694
/* .endpoint = */ endpoint,
695-
/* .name = */ "RPC",
695+
/* .name = */ "RPC[" + std::string(endpoint) + "]",
696696
};
697697

698698
ggml_backend_t backend = new ggml_backend {

0 commit comments

Comments
 (0)