diff --git a/candle_demo/Cargo.lock b/candle_demo/Cargo.lock index afb1964..fcb1424 100644 --- a/candle_demo/Cargo.lock +++ b/candle_demo/Cargo.lock @@ -410,6 +410,27 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cli" +version = "0.1.0" +dependencies = [ + "accelerate-src", + "bindgen_cuda", + "candle-core", + "candle-examples", + "candle-nn", + "candle-transformers", + "clap", + "glm4", + "hf-hub", + "intel-mkl-src", + "owo-colors", + "rand", + "safetensors", + "serde_json", + "tokenizers", +] + [[package]] name = "colorchoice" version = "1.0.2" @@ -1075,27 +1096,6 @@ dependencies = [ "tokenizers", ] -[[package]] -name = "glm4-cli" -version = "0.1.0" -dependencies = [ - "accelerate-src", - "bindgen_cuda", - "candle-core", - "candle-examples", - "candle-nn", - "candle-transformers", - "clap", - "glm4", - "hf-hub", - "intel-mkl-src", - "owo-colors", - "rand", - "safetensors", - "serde_json", - "tokenizers", -] - [[package]] name = "glob" version = "0.3.1" diff --git a/candle_demo/README.org b/candle_demo/README.org new file mode 100644 index 0000000..92e576e --- /dev/null +++ b/candle_demo/README.org @@ -0,0 +1,35 @@ +* candle-GLM-4 +GLM-4-9B 是智谱 AI 推出的最新一代预训练模型 GLM-4 系列中的开源版本。 在语义、数学、推理、代码和知识等多方面的数据集测评中, + + +- [[https://github.com/THUDM/GLM-4][Github]] +- [[https://huggingface.co/THUDM/glm-4-9b][huggingface]] +** Cli +#+begin_src shell + cargo build --release -p cli # Cpu + cargo build --release -p cli --features cuda # if cuda is avalibe + ./target/release/codegeex4-cli --sample-len 500 +#+end_src +** Output_Example +[[../resources/candle_example.png][file:../resources/candle_example.png]] +* Citation +#+begin_src +@misc{glm2024chatglm, + title={ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools}, + author={Team GLM and Aohan Zeng and Bin Xu and Bowen Wang and Chenhui Zhang and Da Yin and Diego Rojas and Guanyu Feng and Hanlin Zhao and Hanyu Lai and Hao Yu and Hongning Wang and Jiadai Sun and Jiajie Zhang and Jiale Cheng and Jiayi Gui and Jie Tang and Jing Zhang and Juanzi Li and Lei Zhao and Lindong Wu and Lucen Zhong and Mingdao Liu and Minlie Huang and Peng Zhang and Qinkai Zheng and Rui Lu and Shuaiqi Duan and Shudan Zhang and Shulin Cao and Shuxun Yang and Weng Lam Tam and Wenyi Zhao and Xiao Liu and Xiao Xia and Xiaohan Zhang and Xiaotao Gu and Xin Lv and Xinghan Liu and Xinyi Liu and Xinyue Yang and Xixuan Song and Xunkai Zhang and Yifan An and Yifan Xu and Yilin Niu and Yuantao Yang and Yueyan Li and Yushi Bai and Yuxiao Dong and Zehan Qi and Zhaoyu Wang and Zhen Yang and Zhengxiao Du and Zhenyu Hou and Zihan Wang}, + year={2024}, + eprint={2406.12793}, + archivePrefix={arXiv}, + primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'} +} +#+end_src +#+begin_src + @misc{wang2023cogvlm, + title={CogVLM: Visual Expert for Pretrained Language Models}, + author={Weihan Wang and Qingsong Lv and Wenmeng Yu and Wenyi Hong and Ji Qi and Yan Wang and Junhui Ji and Zhuoyi Yang and Lei Zhao and Xixuan Song and Jiazheng Xu and Bin Xu and Juanzi Li and Yuxiao Dong and Ming Ding and Jie Tang}, + year={2023}, + eprint={2311.03079}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +#+end_src diff --git a/candle_demo/cli/Cargo.toml b/candle_demo/cli/Cargo.toml index ba20229..ec6a7f8 100644 --- a/candle_demo/cli/Cargo.toml +++ b/candle_demo/cli/Cargo.toml @@ -1,5 +1,5 @@ [package] -name= "glm4-cli" +name= "cli" version.workspace = true edition.workspace = true authors.workspace = true diff --git a/candle_demo/cli/src/main.rs b/candle_demo/cli/src/main.rs index 708130b..cc4cf2a 100755 --- a/candle_demo/cli/src/main.rs +++ b/candle_demo/cli/src/main.rs @@ -27,7 +27,7 @@ fn main() -> Result<(), ()> { ); println!( "temp: {:.2} repeat-penalty: {:.2} repeat-last-n: {}", - args.temperature.unwrap_or(0.95).red(), + args.temperature.unwrap_or(0.6).red(), args.repeat_penalty.red(), args.repeat_last_n.red(), ); @@ -48,7 +48,7 @@ fn main() -> Result<(), ()> { let model_id = match args.model_id { Some(model_id) => model_id.to_string(), - None => "THUDM/glm4-9b".to_string(), + None => "THUDM/glm-4-9b".to_string(), }; let revision = match args.revision { Some(rev) => rev.to_string(), @@ -58,7 +58,7 @@ fn main() -> Result<(), ()> { let tokenizer_filename = match args.tokenizer { Some(file) => std::path::PathBuf::from(file), None => api - .model("THUDM/glm4-9b".to_string()) + .model("THUDM/codegeex4-all-9b".to_string()) .get("tokenizer.json") .unwrap(), }; diff --git a/candle_demo/glm4/src/args.rs b/candle_demo/glm4/src/args.rs index f3bf37d..aa18eb8 100644 --- a/candle_demo/glm4/src/args.rs +++ b/candle_demo/glm4/src/args.rs @@ -18,15 +18,15 @@ pub struct Args { pub temperature: Option, /// Nucleus sampling probability cutoff. - #[arg(long)] - pub top_p: Option, + #[arg(long,default_value_t = 0.6)] + pub top_p: f64, /// The seed to use when generating random samples. #[arg(long)] pub seed: Option, /// The length of the sample to generate (in tokens). - #[arg(long, short = 'n', default_value_t = 5000)] + #[arg(long, short = 'n', default_value_t = 500)] pub sample_len: usize, #[arg(long)] diff --git a/candle_demo/glm4/src/glm4.rs b/candle_demo/glm4/src/glm4.rs index 00469f0..da8965c 100644 --- a/candle_demo/glm4/src/glm4.rs +++ b/candle_demo/glm4/src/glm4.rs @@ -35,7 +35,7 @@ impl Config { ffn_hidden_size: 13696, kv_channels: 128, num_attention_heads: 32, - seq_length: 131072, + seq_length: 8192, layernorm_epsilon: 1e-5, rmsnorm: true, apply_residual_connection_post_layernorm: false, diff --git a/candle_demo/glm4/src/lib.rs b/candle_demo/glm4/src/lib.rs index 9871a1f..09d0bcd 100644 --- a/candle_demo/glm4/src/lib.rs +++ b/candle_demo/glm4/src/lib.rs @@ -28,14 +28,14 @@ impl TextGeneration { tokenizer: Tokenizer, seed: u64, temp: Option, - top_p: Option, + top_p: f64, repeat_penalty: f32, repeat_last_n: usize, verbose_prompt: bool, device: &Device, dtype: DType, ) -> Self { - let logits_processor = LogitsProcessor::new(seed, temp, top_p); + let logits_processor = LogitsProcessor::new(seed, temp, Some(top_p)); Self { model, tokenizer, diff --git a/resources/candle_example.png b/resources/candle_example.png new file mode 100644 index 0000000..ba8efe1 Binary files /dev/null and b/resources/candle_example.png differ