diff --git a/Cargo.toml b/Cargo.toml index 9b3a503..9a1e37d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "agcore" -version = "0.3.6" +version = "0.3.7" edition = "2024" [features] diff --git a/src/prompt/template.rs b/src/prompt/template.rs index 49c3b10..a517481 100644 --- a/src/prompt/template.rs +++ b/src/prompt/template.rs @@ -220,7 +220,7 @@ fn compile_fragments(template: &str) -> Result, PromptError> { fragments.push(Fragment::Literal(literal.clone())); literal.clear(); } - let (tag_content, end) = parse_tag(bytes, i)?; + let (tag_content, end) = parse_tag(template, i)?; i = end; let tag = tag_content.trim(); @@ -252,8 +252,16 @@ fn compile_fragments(template: &str) -> Result, PromptError> { fragments.push(Fragment::Variable { name }); } } else { - literal.push(bytes[i] as char); - i += 1; + debug_assert!(template.is_char_boundary(i)); + match template[i..].chars().next() { + Some(ch) => { + literal.push(ch); + i += ch.len_utf8(); + } + None => { + return Err(PromptError::Parse("模板包含非法字符序列".to_string())); + } + } } } @@ -264,7 +272,8 @@ fn compile_fragments(template: &str) -> Result, PromptError> { Ok(fragments) } -fn parse_tag(bytes: &[u8], start: usize) -> Result<(String, usize), PromptError> { +fn parse_tag(template: &str, start: usize) -> Result<(String, usize), PromptError> { + let bytes = template.as_bytes(); let len = bytes.len(); let mut i = start + 2; let mut content = String::new(); @@ -272,8 +281,16 @@ fn parse_tag(bytes: &[u8], start: usize) -> Result<(String, usize), PromptError> if bytes[i] == b'}' && i + 1 < len && bytes[i + 1] == b'}' { return Ok((content, i + 2)); } - content.push(bytes[i] as char); - i += 1; + debug_assert!(template.is_char_boundary(i)); + match template[i..].chars().next() { + Some(ch) => { + content.push(ch); + i += ch.len_utf8(); + } + None => { + return Err(PromptError::Parse("模板包含非法字符序列".to_string())); + } + } } Err(PromptError::Parse("未闭合的 {{ 标签".to_string())) } @@ -293,7 +310,7 @@ fn parse_block( while i < len && depth > 0 { if bytes[i] == b'{' && i + 1 < len && bytes[i + 1] == b'{' { - let (tag, end) = parse_tag(bytes, i)?; + let (tag, end) = parse_tag(template, i)?; let tag = tag.trim().to_string(); if tag == format!("/{kind}") { depth -= 1; @@ -320,12 +337,20 @@ fn parse_block( i = end; } } else { - if is_else { - else_body.push(bytes[i] as char); - } else { - body.push(bytes[i] as char); + debug_assert!(template.is_char_boundary(i)); + match template[i..].chars().next() { + Some(ch) => { + if is_else { + else_body.push(ch); + } else { + body.push(ch); + } + i += ch.len_utf8(); + } + None => { + return Err(PromptError::Parse("模板包含非法字符序列".to_string())); + } } - i += 1; } } @@ -341,7 +366,7 @@ fn parse_each_block(template: &str, start: usize) -> Result<(Vec, usiz while i < len && depth > 0 { if bytes[i] == b'{' && i + 1 < len && bytes[i + 1] == b'{' { - let (tag, end) = parse_tag(bytes, i)?; + let (tag, end) = parse_tag(template, i)?; let tag = tag.trim().to_string(); if tag == "/each" { depth -= 1; @@ -361,8 +386,16 @@ fn parse_each_block(template: &str, start: usize) -> Result<(Vec, usiz i = end; } } else { - body.push(bytes[i] as char); - i += 1; + debug_assert!(template.is_char_boundary(i)); + match template[i..].chars().next() { + Some(ch) => { + body.push(ch); + i += ch.len_utf8(); + } + None => { + return Err(PromptError::Parse("模板包含非法字符序列".to_string())); + } + } } } @@ -377,7 +410,7 @@ fn parse_raw_block(template: &str, start: usize) -> Result<(String, usize), Prom while i < len { if bytes[i] == b'{' && i + 1 < len && bytes[i + 1] == b'{' { - let (tag, end) = parse_tag(bytes, i)?; + let (tag, end) = parse_tag(template, i)?; let tag = tag.trim().to_string(); if tag == "/raw" { return Ok((content, end)); @@ -386,8 +419,16 @@ fn parse_raw_block(template: &str, start: usize) -> Result<(String, usize), Prom i = end; } } else { - content.push(bytes[i] as char); - i += 1; + debug_assert!(template.is_char_boundary(i)); + match template[i..].chars().next() { + Some(ch) => { + content.push(ch); + i += ch.len_utf8(); + } + None => { + return Err(PromptError::Parse("模板包含非法字符序列".to_string())); + } + } } } @@ -528,3 +569,170 @@ impl PromptTemplateRegistry { tpl.render(ctx) } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn preserves_chinese_literal() -> Result<(), PromptError> { + let source = "这是一个纯中文模板。"; + let template = PromptTemplate::compile(source)?; + + assert_eq!(template.render(&TemplateContext::new())?, source); + Ok(()) + } + + #[test] + fn preserves_real_world_failure_text() -> Result<(), PromptError> { + let source = "你是采集策略专家,负责审查已采集的产品编码结果,决策下一轮搜索方向。★ 下一步→「严格校验」,使用全角标点:,;!"; + let template = PromptTemplate::compile(source)?; + + assert_eq!(template.render(&TemplateContext::new())?, source); + Ok(()) + } + + #[test] + fn renders_chinese_with_variable() -> Result<(), PromptError> { + let template = PromptTemplate::compile("你好,{{ name }}!")?; + let mut ctx = TemplateContext::new(); + ctx.insert("name", "小明"); + + assert_eq!(template.render(&ctx)?, "你好,小明!"); + Ok(()) + } + + #[test] + fn preserves_chinese_in_if_body() -> Result<(), PromptError> { + let template = PromptTemplate::compile("{{#if enabled}}已启用{{/if}}")?; + let mut ctx = TemplateContext::new(); + ctx.insert("enabled", true); + + assert_eq!(template.render(&ctx)?, "已启用"); + Ok(()) + } + + #[test] + fn preserves_chinese_in_else_body() -> Result<(), PromptError> { + let template = PromptTemplate::compile("{{#if enabled}}已启用{{else}}未启用{{/if}}")?; + let mut ctx = TemplateContext::new(); + ctx.insert("enabled", false); + + assert_eq!(template.render(&ctx)?, "未启用"); + Ok(()) + } + + #[test] + fn preserves_chinese_in_each_body() -> Result<(), PromptError> { + let template = PromptTemplate::compile("{{#each items}}项目:{{item}};{{/each}}")?; + let ctx = TemplateContext::from_json(&json!({"items": ["甲", "乙"]}))?; + + assert_eq!(template.render(&ctx)?, "项目:甲;项目:乙;"); + Ok(()) + } + + #[test] + fn preserves_chinese_in_raw_body() -> Result<(), PromptError> { + let template = PromptTemplate::compile("{{#raw}}原始中文:{{name}}{{/raw}}")?; + + assert_eq!( + template.render(&TemplateContext::new())?, + "原始中文:{{name}}" + ); + Ok(()) + } + + #[test] + fn preserves_four_byte_characters() -> Result<(), PromptError> { + let source = "你好👋🌍"; + let template = PromptTemplate::compile(source)?; + + assert_eq!(template.render(&TemplateContext::new())?, source); + Ok(()) + } + + #[test] + fn parses_multibyte_characters_next_to_tag_boundaries() -> Result<(), PromptError> { + let template = PromptTemplate::compile("前{{name}}后")?; + let mut ctx = TemplateContext::new(); + ctx.insert("name", "中"); + + assert_eq!(template.render(&ctx)?, "前中后"); + Ok(()) + } + + #[test] + fn preserves_chinese_at_end_of_template() -> Result<(), PromptError> { + let source = "template ends with 中文"; + let template = PromptTemplate::compile(source)?; + + assert_eq!(template.render(&TemplateContext::new())?, source); + Ok(()) + } + + #[test] + fn renders_empty_template() -> Result<(), PromptError> { + let template = PromptTemplate::compile("")?; + + assert_eq!(template.render(&TemplateContext::new())?, ""); + Ok(()) + } + + #[test] + fn rejects_unclosed_tag_after_chinese_without_panicking() { + assert!(matches!( + PromptTemplate::compile("中文{{未闭合"), + Err(PromptError::Parse(_)) + )); + } + + #[test] + fn preserves_ascii_template_behavior() -> Result<(), PromptError> { + let template = PromptTemplate::compile( + "Hello {{name}}! {{#if active}}Active{{else}}Inactive{{/if}} {{#each items}}[{{item}}]{{/each}} {{#raw}}{{raw}}{{/raw}}", + )?; + let ctx = TemplateContext::from_json(&json!({ + "name": "Alice", + "active": true, + "items": ["a", "b"] + }))?; + + assert_eq!(template.render(&ctx)?, "Hello Alice! Active [a][b] {{raw}}"); + Ok(()) + } + + #[test] + fn renders_chinese_variable_name() -> Result<(), PromptError> { + let template = PromptTemplate::compile("{{ 问候 }},世界!")?; + let mut ctx = TemplateContext::new(); + ctx.insert("问候", "你好"); + + assert_eq!(template.render(&ctx)?, "你好,世界!"); + Ok(()) + } + + #[test] + fn renders_chinese_if_condition() -> Result<(), PromptError> { + let template = PromptTemplate::compile("{{#if 已启用}}条件成立{{/if}}")?; + let mut ctx = TemplateContext::new(); + ctx.insert("已启用", true); + + assert_eq!(template.render(&ctx)?, "条件成立"); + Ok(()) + } + + #[test] + fn preserves_chinese_in_nested_if_and_each_blocks() -> Result<(), PromptError> { + let template = PromptTemplate::compile( + "{{#if 已启用}}列表:{{#each 项目}}【{{item}}】{{/each}}{{/if}}", + )?; + let ctx = TemplateContext::from_json(&json!({ + "已启用": true, + "项目": ["甲", "乙"] + }))?; + + assert_eq!(template.render(&ctx)?, "列表:【甲】【乙】"); + Ok(()) + } +}