echodict/llama.cpp

1MAKEFLAGS += --no-print-directory2 3define validate_model_path4	@if [ -z "$(MODEL_PATH)" ]; then \5		echo "Error: MODEL_PATH must be provided either as:"; \6		echo "  1. Environment variable: export MODEL_PATH=/path/to/model"; \7		echo "  2. Command line argument: make $(1) MODEL_PATH=/path/to/model"; \8		exit 1; \9	fi10endef11 12define validate_embedding_model_path13	@if [ -z "$(EMBEDDING_MODEL_PATH)" ]; then \14		echo "Error: EMBEDDING_MODEL_PATH must be provided either as:"; \15		echo "  1. Environment variable: export EMBEDDING_MODEL_PATH=/path/to/model"; \16		echo "  2. Command line argument: make $(1) EMBEDDING_MODEL_PATH=/path/to/model"; \17		exit 1; \18	fi19endef20 21define quantize_model22	@CONVERTED_MODEL="$(1)" QUANTIZED_TYPE="$(QUANTIZED_TYPE)" \23	TOKEN_EMBD_TYPE="$(TOKEN_EMBD_TYPE)" OUTPUT_TYPE="$(OUTPUT_TYPE)" \24	./scripts/utils/quantize.sh "$(1)" "$(QUANTIZED_TYPE)" "$(TOKEN_EMBD_TYPE)" "$(OUTPUT_TYPE)"25	@echo "Export the quantized model path to $(2) variable in your environment"26endef27 28DEVICE ?= auto29 30###31### Casual Model targets/recipes32###33causal-convert-model-bf16: OUTTYPE=bf1634causal-convert-model-bf16: causal-convert-model35 36causal-convert-model-debug: DEBUG=--debug37causal-convert-model-debug: causal-convert-model38 39causal-convert-model:40	$(call validate_model_path,causal-convert-model)41	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \42	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \43	./scripts/causal/convert-model.sh $(DEBUG)44 45causal-convert-mm-model-bf16: OUTTYPE=bf1646causal-convert-mm-model-bf16: MM_OUTTYPE=f1647causal-convert-mm-model-bf16: causal-convert-mm-model48 49causal-convert-mm-model:50	$(call validate_model_path,causal-convert-mm-model)51	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \52	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \53	./scripts/causal/convert-model.sh54 55	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(MM_OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \56	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \57	./scripts/causal/convert-model.sh --mmproj58 59causal-run-original-model:60	$(call validate_model_path,causal-run-original-model)61	@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/run-org-model.py --device "$(DEVICE)"62 63causal-run-converted-model:64	@CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/causal/run-converted-model.sh65 66causal-verify-logits: causal-run-original-model causal-run-converted-model67	@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/compare-logits.py68	@MODEL_PATH="$(MODEL_PATH)" ./scripts/utils/check-nmse.py -m ${MODEL_PATH}69 70causal-run-original-embeddings:71	@./scripts/causal/run-casual-gen-embeddings-org.py72 73causal-run-converted-embeddings:74	@./scripts/causal/run-converted-model-embeddings-logits.sh75 76causal-verify-embeddings: causal-run-original-embeddings causal-run-converted-embeddings77	@./scripts/causal/compare-embeddings-logits.sh78 79causal-inspect-original-model:80	@./scripts/utils/inspect-org-model.py --list-all -s81 82causal-list-original-model-tensors:83	@./scripts/utils/inspect-org-model.py --list-all-short -s84 85causal-inspect-converted-model:86	@./scripts/utils/inspect-converted-model.sh87 88causal-start-embedding-server:89	@./scripts/utils/run-embedding-server.sh ${CONVERTED_MODEL}90 91causal-curl-embedding-endpoint: causal-run-original-embeddings92	@./scripts/utils/curl-embedding-server.sh | ./scripts/causal/compare-embeddings-logits.sh93 94causal-quantize-Q8_0: QUANTIZED_TYPE = Q8_095causal-quantize-Q8_0: causal-quantize-model96 97causal-quantize-Q4_0: QUANTIZED_TYPE = Q4_098causal-quantize-Q4_0: causal-quantize-model99 100# For Quantization Aware Trained (QAT) models in Q4_0 we explicitly set the101# token embedding and output types to Q8_0 instead of the default Q6_K.102causal-quantize-qat-Q4_0: QUANTIZED_TYPE = Q4_0103causal-quantize-qat-Q4_0: TOKEN_EMBD_TYPE = Q8_0104causal-quantize-qat-Q4_0: OUTPUT_TYPE = Q8_0105causal-quantize-qat-Q4_0: causal-quantize-model106 107causal-quantize-model:108	$(call quantize_model,$(CONVERTED_MODEL),QUANTIZED_MODEL)109 110causal-run-quantized-model:111	@QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/causal/run-converted-model.sh ${QUANTIZED_MODEL}112 113 114###115### Embedding Model targets/recipes116###117 118embedding-convert-model-bf16: OUTTYPE=bf16119embedding-convert-model-bf16: embedding-convert-model120 121embedding-convert-model:122	$(call validate_embedding_model_path,embedding-convert-model)123	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \124	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \125	./scripts/embedding/convert-model.sh126 127embedding-convert-model-st:128	$(call validate_embedding_model_path,embedding-convert-model-st)129	@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \130	METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \131	./scripts/embedding/convert-model.sh -st132 133embedding-run-original-model:134	$(call validate_embedding_model_path,embedding-run-original-model)135	@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" \136	USE_SENTENCE_TRANSFORMERS="$(USE_SENTENCE_TRANSFORMERS)" \137	./scripts/embedding/run-original-model.py \138	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)") \139	$(if $(USE_SENTENCE_TRANSFORMERS),--use-sentence-transformers)140 141embedding-run-original-model-st: USE_SENTENCE_TRANSFORMERS=1142embedding-run-original-model-st: embedding-run-original-model143 144embedding-run-converted-model:145	@./scripts/embedding/run-converted-model.sh $(CONVERTED_EMBEDDING_MODEL) \146	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)") \147	$(if $(EMBD_NORMALIZE),--embd-normalize "$(EMBD_NORMALIZE)")148 149embedding-verify-logits: embedding-run-original-model embedding-run-converted-model150	@./scripts/embedding/compare-embeddings-logits.sh \151	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)")152 153embedding-verify-logits-st: embedding-run-original-model-st embedding-run-converted-model154	@./scripts/embedding/compare-embeddings-logits.sh \155	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)")156 157embedding-inspect-original-model:158	$(call validate_embedding_model_path,embedding-inspect-original-model)159	@EMBEDDING_MODEL_PATH="$(EMBEDDING_MODEL_PATH)" ./scripts/utils/inspect-org-model.py -m ${EMBEDDING_MODEL_PATH} --list-all -s160 161embedding-inspect-converted-model:162	@CONVERTED_EMBEDDING_MODEL="$(CONVERTED_EMBEDDING_MODEL)" ./scripts/utils/inspect-converted-model.sh ${CONVERTED_EMBEDDING_MODEL}163 164embedding-start-embedding-server:165	@./scripts/utils/run-embedding-server.sh ${CONVERTED_EMBEDDING_MODEL}166 167embedding-curl-embedding-endpoint:168	@./scripts/utils/curl-embedding-server.sh | ./scripts/embedding/compare-embeddings-logits.sh169 170embedding-quantize-Q8_0: QUANTIZED_TYPE = Q8_0171embedding-quantize-Q8_0: embedding-quantize-model172 173embedding-quantize-Q4_0: QUANTIZED_TYPE = Q4_0174embedding-quantize-Q4_0: embedding-quantize-model175 176# For Quantization Aware Trained (QAT) models in Q4_0 we explicitly set the177# token embedding and output types to Q8_0 instead of the default Q6_K.178embedding-quantize-qat-Q4_0: QUANTIZED_TYPE = Q4_0179embedding-quantize-qat-Q4_0: TOKEN_EMBD_TYPE = Q8_0180embedding-quantize-qat-Q4_0: OUTPUT_TYPE = Q8_0181embedding-quantize-qat-Q4_0: embedding-quantize-model182 183embedding-quantize-model:184	$(call quantize_model,$(CONVERTED_EMBEDDING_MODEL),QUANTIZED_EMBEDDING_MODEL)185 186embedding-run-quantized-model:187	@./scripts/embedding/run-converted-model.sh $(QUANTIZED_EMBEDDING_MODEL) \188	$(if $(PROMPTS_FILE),--prompts-file "$(PROMPTS_FILE)")189 190###191### Perplexity targets/recipes192###193perplexity-data-gen:194	CONVERTED_MODEL="$(CONVERTED_MODEL)" ./scripts/utils/perplexity-gen.sh195 196perplexity-run-full:197	QUANTIZED_MODEL="$(QUANTIZED_MODEL)" LOOGITS_FILE="$(LOGITS_FILE)" \198	./scripts/utils/perplexity-run.sh199 200perplexity-run:201	QUANTIZED_MODEL="$(QUANTIZED_MODEL)" ./scripts/utils/perplexity-run-simple.sh202 203###204### HuggingFace targets/recipes205###206 207hf-create-model:208	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}"209 210hf-create-model-dry-run:211	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -d212 213hf-create-model-embedding:214	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -e215 216hf-create-model-embedding-dry-run:217	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -e -d218 219hf-create-model-private:220	@./scripts/utils/hf-create-model.py -m "${MODEL_NAME}" -ns "${NAMESPACE}" -b "${ORIGINAL_BASE_MODEL}" -p221 222hf-upload-gguf-to-model:223	@./scripts/utils/hf-upload-gguf-model.py -m "${MODEL_PATH}" -r "${REPO_ID}" -o "${NAME_IN_REPO}"224 225hf-create-collection:226	@./scripts/utils/hf-create-collection.py -n "${NAME}" -d "${DESCRIPTION}" -ns "${NAMESPACE}"227 228hf-add-model-to-collection:229	@./scripts/utils/hf-add-model-to-collection.py -c "${COLLECTION}" -m "${MODEL}"230 231 232.PHONY: clean233clean:234	@${RM} -rf data .converted_embedding_model.txt .converted_model.txt .embedding_model_name.txt .model_name.txt235 236