Skip to content

Commit 542ae3b

Browse files
0cc4m0cc4m
committed
Merge upstream changes, fix conflicts
2 parents 918c333 + 52531fd commit 542ae3b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+6768
-5342
lines changed

‎.devops/nix/jetson-support.nix

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88
pkgsCuda,
99
...
1010
}:
11-
lib.optionalAttrs (system == "aarch64-linux") {
12-
packages =
11+
{
12+
legacyPackages =
1313
let
14-
caps.jetson-xavier = "7.2";
15-
caps.jetson-orin = "8.7";
16-
caps.jetson-nano = "5.3";
14+
caps.llamaPackagesXavier = "7.2";
15+
caps.llamaPackagesOrin = "8.7";
16+
caps.llamaPackagesTX2 = "6.2";
17+
caps.llamaPackagesNano = "5.3";
1718

1819
pkgsFor =
1920
cap:
@@ -27,6 +28,12 @@
2728
};
2829
};
2930
in
30-
builtins.mapAttrs (name: cap: ((pkgsFor cap).callPackage ./scope.nix { }).llama-cpp) caps;
31+
builtins.mapAttrs (name: cap: (pkgsFor cap).callPackage ./scope.nix { }) caps;
32+
33+
packages = lib.optionalAttrs (system == "aarch64-linux") {
34+
jetson-xavier = config.legacyPackages.llamaPackagesXavier.llama-cpp;
35+
jetson-orin = config.legacyPackages.llamaPackagesOrin.llama-cpp;
36+
jetson-nano = config.legacyPackages.llamaPackagesNano.llama-cpp;
37+
};
3138
};
3239
}

‎.devops/nix/package.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
git,
1010
python3,
1111
mpi,
12-
openblas, # TODO: Use the generic `blas` so users could switch betwen alternative implementations
12+
openblas, # TODO: Use the generic `blas` so users could switch between alternative implementations
1313
cudaPackages,
1414
darwin,
1515
rocmPackages,

‎.github/workflows/build.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -515,7 +515,6 @@ jobs:
515515
- name: Build Xcode project
516516
run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build
517517

518-
519518
# freeBSD-latest:
520519
# runs-on: macos-12
521520
# steps:

‎.github/workflows/nix-ci.yml

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Nix CI
2+
3+
on:
4+
workflow_dispatch: # allows manual triggering
5+
push:
6+
branches:
7+
- master
8+
paths: ['.github/workflows/**', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', '**/*.sh', '**/*.py', '**/*.nix']
9+
pull_request:
10+
types: [opened, synchronize, reopened]
11+
paths: ['**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', '**/*.sh', '**/*.py', '**/*.nix']
12+
13+
jobs:
14+
nix-eval:
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
os: [ ubuntu-latest, macos-latest ]
19+
runs-on: ${{ matrix.os }}
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v4
23+
- name: Install Nix
24+
uses: DeterminateSystems/nix-installer-action@v9
25+
with:
26+
github-token: ${{ secrets.GITHUB_TOKEN }}
27+
extra-conf: |
28+
extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org
29+
extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=
30+
- uses: DeterminateSystems/magic-nix-cache-action@v2
31+
with:
32+
upstream-cache: https://${{ matrix.cachixName }}.cachix.org
33+
- name: List all flake outputs
34+
run: nix flake show --all-systems
35+
- name: Show all output paths
36+
run: >
37+
nix run github:nix-community/nix-eval-jobs
38+
-- --gc-roots-dir gcroot
39+
--flake
40+
".#packages.$(nix eval --raw --impure --expr builtins.currentSystem)"
41+
nix-build:
42+
if: ${{ vars.CACHIX_NAME != '' }}
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
os: [ ubuntu-latest, macos-latest ]
47+
runs-on: ${{ matrix.os }}
48+
steps:
49+
- name: Checkout repository
50+
uses: actions/checkout@v4
51+
- name: Install Nix
52+
uses: DeterminateSystems/nix-installer-action@v9
53+
with:
54+
github-token: ${{ secrets.GITHUB_TOKEN }}
55+
extra-conf: |
56+
extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org
57+
extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=
58+
- uses: DeterminateSystems/magic-nix-cache-action@v2
59+
with:
60+
upstream-cache: https://${{ matrix.cachixName }}.cachix.org
61+
- name: Set-up cachix to push the results to
62+
uses: cachix/cachix-action@v13
63+
with:
64+
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
65+
name: ${{ vars.CACHIX_NAME }}
66+
- name: Build
67+
run: >
68+
nix run github:Mic92/nix-fast-build
69+
-- --skip-cached --no-nom
70+
--flake
71+
".#checks.$(nix eval --raw --impure --expr builtins.currentSystem)"
72+
nix-build-aarch64:
73+
if: ${{ vars.CACHIX_NAME != '' }}
74+
runs-on: ubuntu-latest
75+
steps:
76+
- name: Checkout repository
77+
uses: actions/checkout@v4
78+
- name: Install QEMU
79+
# Copy-paste from https://github.com/orgs/community/discussions/8305#discussioncomment-5888654
80+
run: |
81+
sudo apt-get install -y qemu-user-static qemu-system-aarch64
82+
sudo usermod -a -G kvm $USER
83+
- name: Install Nix
84+
uses: DeterminateSystems/nix-installer-action@v9
85+
with:
86+
github-token: ${{ secrets.GITHUB_TOKEN }}
87+
extra-conf: |
88+
extra-platforms = aarch64-linux
89+
extra-system-features = nixos-test kvm
90+
extra-substituters = https://${{ vars.CACHIX_NAME }}.cachix.org https://cuda-maintainers.cachix.org
91+
extra-trusted-public-keys = ${{ vars.CACHIX_PUBLIC_KEY }} cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=
92+
- uses: DeterminateSystems/magic-nix-cache-action@v2
93+
with:
94+
upstream-cache: https://${{ matrix.cachixName }}.cachix.org
95+
- name: Set-up cachix to push the results to
96+
uses: cachix/cachix-action@v13
97+
with:
98+
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
99+
name: ${{ vars.CACHIX_NAME }}
100+
- name: Show all output paths
101+
run: >
102+
nix run github:nix-community/nix-eval-jobs
103+
-- --gc-roots-dir gcroot
104+
--flake
105+
".#packages.aarch64-linux"
106+
- name: Build
107+
run: >
108+
nix run github:Mic92/nix-fast-build
109+
-- --skip-cached --no-nom
110+
--systems aarch64-linux
111+
--flake
112+
".#checks.aarch64-linux"

‎.github/workflows/nix-flake-update.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: update-flake-lock
2+
on:
3+
workflow_dispatch:
4+
schedule:
5+
- cron: '0 0 * * 0' # runs weekly on Sunday at 00:00
6+
7+
jobs:
8+
lockfile:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- name: Checkout repository
12+
uses: actions/checkout@v4
13+
- name: Install Nix
14+
uses: DeterminateSystems/nix-installer-action@main
15+
- name: Update flake.lock
16+
uses: DeterminateSystems/update-flake-lock@main
17+
with:
18+
pr-title: "nix: update flake.lock"
19+
pr-labels: |
20+
nix
21+
pr-reviewers: philiptaron,SomeoneSerge
22+
token: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/nix-flakestry.yml

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.github/workflows/nix-publish-flake.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Make the flake discoverable on https://flakestry.dev and https://flakehub.com/flakes
2+
name: "Publish a flake to flakestry & flakehub"
3+
on:
4+
push:
5+
tags:
6+
- "*"
7+
workflow_dispatch:
8+
inputs:
9+
tag:
10+
description: "The existing tag to publish"
11+
type: "string"
12+
required: true
13+
jobs:
14+
flakestry-publish:
15+
runs-on: ubuntu-latest
16+
permissions:
17+
id-token: "write"
18+
contents: "read"
19+
steps:
20+
- uses: flakestry/flakestry-publish@main
21+
with:
22+
version: "${{ inputs.tag || github.ref_name }}"
23+
flakehub-publish:
24+
runs-on: "ubuntu-latest"
25+
permissions:
26+
id-token: "write"
27+
contents: "read"
28+
steps:
29+
- uses: "actions/checkout@v4"
30+
with:
31+
ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
32+
- uses: "DeterminateSystems/nix-installer-action@main"
33+
- uses: "DeterminateSystems/flakehub-push@main"
34+
with:
35+
visibility: "public"
36+
tag: "${{ inputs.tag }}"

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ models-mnt
5151
/lookup
5252
/main
5353
/metal
54+
/passkey
5455
/perplexity
5556
/q8dot
5657
/quantize

‎CMakeLists.txt

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ option(LLAMA_CLBLAST "llama: use CLBlast"
9696
option(LLAMA_VULKAN "llama: use Vulkan" OFF)
9797
option(LLAMA_METAL "llama: use Metal" ${LLAMA_METAL_DEFAULT})
9898
option(LLAMA_METAL_NDEBUG "llama: disable Metal debugging" OFF)
99+
option(LLAMA_METAL_SHADER_DEBUG "llama: compile Metal with -fno-fast-math" OFF)
99100
option(LLAMA_MPI "llama: use MPI" OFF)
100101
option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF)
101102

@@ -155,9 +156,9 @@ if (APPLE AND LLAMA_ACCELERATE)
155156
endif()
156157

157158
if (LLAMA_METAL)
158-
find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
159-
find_library(METAL_FRAMEWORK Metal REQUIRED)
160-
find_library(METALKIT_FRAMEWORK MetalKit REQUIRED)
159+
find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
160+
find_library(METAL_FRAMEWORK Metal REQUIRED)
161+
find_library(METALKIT_FRAMEWORK MetalKit REQUIRED)
161162

162163
message(STATUS "Metal framework found")
163164
set(GGML_HEADERS_METAL ggml-metal.h)
@@ -174,6 +175,35 @@ if (LLAMA_METAL)
174175
# copy ggml-metal.metal to bin directory
175176
configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
176177

178+
if (LLAMA_METAL_SHADER_DEBUG)
179+
# custom command to do the following:
180+
# xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
181+
# xcrun -sdk macosx metallib ggml-metal.air -o default.metallib
182+
#
183+
# note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works
184+
# disabling fast math is needed in order to pass tests/test-backend-ops
185+
# note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1
186+
# note: unfortunately, we have to call it default.metallib instead of ggml.metallib
187+
# ref: https://github.com/ggerganov/whisper.cpp/issues/1720
188+
set(XC_FLAGS -fno-fast-math -fno-inline -g)
189+
if (LLAMA_QKK_64)
190+
set(XC_FLAGS ${XC_FLAGS} -DQK_K=64)
191+
endif()
192+
193+
add_custom_command(
194+
OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
195+
COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
196+
COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
197+
DEPENDS ggml-metal.metal
198+
COMMENT "Compiling Metal kernels"
199+
)
200+
201+
add_custom_target(
202+
ggml-metal ALL
203+
DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
204+
)
205+
endif()
206+
177207
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS}
178208
${FOUNDATION_LIBRARY}
179209
${METAL_FRAMEWORK}
@@ -201,7 +231,11 @@ if (LLAMA_BLAS)
201231
if (${LLAMA_BLAS_VENDOR} MATCHES "Generic")
202232
pkg_check_modules(DepBLAS REQUIRED blas)
203233
elseif (${LLAMA_BLAS_VENDOR} MATCHES "OpenBLAS")
204-
pkg_check_modules(DepBLAS REQUIRED openblas)
234+
# As of openblas v0.3.22, the 64-bit is named openblas64.pc
235+
pkg_check_modules(DepBLAS openblas64)
236+
if (NOT DepBLAS_FOUND)
237+
pkg_check_modules(DepBLAS REQUIRED openblas)
238+
endif()
205239
elseif (${LLAMA_BLAS_VENDOR} MATCHES "FLAME")
206240
pkg_check_modules(DepBLAS REQUIRED blis)
207241
elseif (${LLAMA_BLAS_VENDOR} MATCHES "ATLAS")

‎Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
BUILD_TARGETS = \
33
main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \
44
simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search \
5-
speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup tests/test-c.o
5+
speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey tests/test-c.o
66

77
# Binaries only useful for tests
88
TEST_TARGETS = \
@@ -680,6 +680,9 @@ lookahead: examples/lookahead/lookahead.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS
680680
lookup: examples/lookup/lookup.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
681681
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
682682

683+
passkey: examples/passkey/passkey.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
684+
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
685+
683686
ifdef LLAMA_METAL
684687
metal: examples/metal/metal.cpp ggml.o $(OBJS)
685688
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)

‎Package.swift

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,17 @@ let package = Package(
1313
products: [
1414
.library(name: "llama", targets: ["llama"]),
1515
],
16+
dependencies: [
17+
.package(url: "https://github.com/ggerganov/ggml.git", .branch("master"))
18+
],
1619
targets: [
1720
.target(
1821
name: "llama",
22+
dependencies: ["ggml"],
1923
path: ".",
2024
exclude: [],
2125
sources: [
22-
"ggml.c",
2326
"llama.cpp",
24-
"ggml-alloc.c",
25-
"ggml-backend.c",
26-
"ggml-quants.c",
27-
"ggml-metal.m",
28-
],
29-
resources: [
30-
.process("ggml-metal.metal")
3127
],
3228
publicHeadersPath: "spm-headers",
3329
cSettings: [

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ as the main playground for developing new features for the [ggml](https://github
118118
- Python: [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python)
119119
- Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp)
120120
- Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp)
121+
- JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp)
121122
- Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb)
122123
- Rust: [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp)
123124
- C#/.NET: [SciSharp/LLamaSharp](https://github.com/SciSharp/LLamaSharp)

‎awq-py/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
torch>=2.0.0
1+
torch>=2.1.1
22
transformers>=4.32.0

0 commit comments

Comments
 (0)