diff --git a/agents/bengali-reviewer.md b/agents/bengali-reviewer.md index 2e03fe64c..48734338b 100644 --- a/agents/bengali-reviewer.md +++ b/agents/bengali-reviewer.md @@ -1,7 +1,7 @@ --- name: bengali-reviewer description: Reviews code handling Bengali (Bangla) text for Unicode correctness, proper normalization, script-aware processing, and internationalization best practices. Use when code processes, displays, or stores Bengali text. -tools: Read, Grep, Glob, Bash +tools: Read, Grep, Glob model: sonnet --- @@ -24,7 +24,7 @@ When invoked: 2. **Check Unicode correctness** — Verify NFC normalization, correct handling of conjunct consonants (যুক্তবর্ণ), vowel signs (কার), and Hasanta (্). 3. **Validate text processing** — Check tokenization, search, sort, and comparison operations for Bengali-awareness. 4. **Review rendering and display** — Ensure proper font support, line breaking, and input handling for Bengali script. -5. **Check data layer** — Verify database columns use UTF-8/UTF-8MB4, API responses include proper charset headers, and file I/O specifies encoding. +5. **Check data layer** — Verify database columns use Unicode-capable encoding (utf8mb4 for MySQL, UTF-8 for PostgreSQL/SQLite), API responses include proper charset headers (`charset=utf-8`), and file I/O specifies encoding. 6. **Report findings** — Use the output format below. Only report issues you are confident about (>80% sure it is a real problem). ## Confidence-Based Filtering @@ -91,7 +91,9 @@ zero rows and verdict `APPROVE`. - **Missing NFC normalization before storage** — Bengali text stored without normalization will cause duplicate records and failed lookups. - **Non-Unicode database encoding** — MySQL columns using `latin1` or `ascii` - charset for Bengali text will silently corrupt or truncate data. + charset instead of `utf8mb4` for Bengali text will silently corrupt or + truncate data. For PostgreSQL, verify UTF-8 database encoding. For SQLite, + ensure text is handled as UTF-8. - **File I/O without encoding** — Reading/writing Bengali text files without explicit UTF-8 encoding on platforms where the default is not UTF-8. @@ -130,9 +132,11 @@ if re.match(r'^[a-zA-Z\u0980-\u09FF]+$', name): ### MEDIUM — Quality Issues -- **Zero-width characters not stripped** — Web-scraped Bengali text often contains - ZWNJ (U+200C), ZWJ (U+200D), and other invisible characters that cause - matching failures if not cleaned. +- **Unwanted zero-width characters** — Web-scraped Bengali text often contains + invisible characters like ZWSP (U+200B) and BOM (U+FEFF) that cause matching + failures. Note: preserve ZWNJ (U+200C) and ZWJ (U+200D) by default as they + are orthographically significant for Bengali conjuncts — only remove them when + an explicit input policy identifies the sequence as invalid. - **Hardcoded strings** — Bengali UI text embedded in source code instead of i18n resource files. - **Missing locale in formatting** — Dates, numbers, and currencies displayed @@ -167,21 +171,25 @@ Fix: Use grapheme cluster segmentation via `regex.findall(r"\X", text)` or ### Summary Format -End every review with: +End every review with a summary table. Calculate each severity count from your +actual findings and derive the verdict accordingly. Example format: ``` ## Bengali Text Review Summary | Severity | Count | Status | |----------|-------|--------| -| CRITICAL | 0 | pass | -| HIGH | 2 | warn | -| MEDIUM | 1 | info | -| LOW | 1 | note | +| CRITICAL | | pass/fail | +| HIGH | | pass/warn | +| MEDIUM | | pass/info | +| LOW | | pass/note | -Verdict: WARNING — 2 HIGH issues should be resolved before merge. +Verdict: APPROVE / WARNING / BLOCK — ``` +A clean review with zero findings across all severities is valid — use +verdict `APPROVE` with status `pass` for all rows. + ## Approval Criteria - **Approve**: No CRITICAL or HIGH issues, including clean reviews with zero diff --git a/docs/bn/README.md b/docs/bn/README.md index feb4771ea..c5c2cb216 100644 --- a/docs/bn/README.md +++ b/docs/bn/README.md @@ -1,4 +1,4 @@ -**ভাষা:** [English](../../README.md) | [Português (Brasil)](../pt-BR/README.md) | [简体中文](../../README.zh-CN.md) | [繁體中文](../zh-TW/README.md) | [日本語](../ja-JP/README.md) | [한국어](../ko-KR/README.md) | [Türkçe](../tr/README.md) | [Русский](../ru/README.md) | [Tiếng Việt](../vi-VN/README.md) | [ไทย](../th/README.md) | [Deutsch](../de-DE/README.md) | বাংলা +**ভাষা:** [English](../../README.md) | [Português (Brasil)](../pt-BR/README.md) | [简体中文](../../README.zh-CN.md) | [繁體中文](../zh-TW/README.md) | [日本語](../ja-JP/README.md) | [한국어](../ko-KR/README.md) | [Türkçe](../tr/README.md) | [Русский](../ru/README.md) | [Tiếng Việt](../vi-VN/README.md) | [ไทย](../th/README.md) | [Deutsch](../de-DE/README.md) | [Español](../es/README.md) | বাংলা # Everything Claude Code @@ -15,7 +15,7 @@ **Language / ভাষা / 语言 / 語言 / 언어 / Dil / Язык / Ngôn ngữ** -[**English**](../../README.md) | [Português (Brasil)](../pt-BR/README.md) | [简体中文](../../README.zh-CN.md) | [繁體中文](../zh-TW/README.md) | [日本語](../ja-JP/README.md) | [한국어](../ko-KR/README.md) | [Türkçe](../tr/README.md) | [Русский](../ru/README.md) | [Tiếng Việt](../vi-VN/README.md) | [ไทย](../th/README.md) | [Deutsch](../de-DE/README.md) | [বাংলা](README.md) +[**English**](../../README.md) | [Português (Brasil)](../pt-BR/README.md) | [简体中文](../../README.zh-CN.md) | [繁體中文](../zh-TW/README.md) | [日本語](../ja-JP/README.md) | [한국어](../ko-KR/README.md) | [Türkçe](../tr/README.md) | [Русский](../ru/README.md) | [Tiếng Việt](../vi-VN/README.md) | [ไทย](../th/README.md) | [Deutsch](../de-DE/README.md) | [Español](../es/README.md) | [বাংলা](README.md) diff --git a/skills/bengali-nlp/SKILL.md b/skills/bengali-nlp/SKILL.md index 020e785f9..8a3dd4fd1 100644 --- a/skills/bengali-nlp/SKILL.md +++ b/skills/bengali-nlp/SKILL.md @@ -27,8 +27,8 @@ Bengali script occupies U+0980–U+09FF. Key character classes: ``` U+0981–U+0983 Chandrabindu, Anusvara, Visarga -U+0985–U+0990 Independent vowels (অ আ ই ঈ উ ঊ ঋ এ ঐ) -U+0993–U+09B0 Consonants (ও–র) +U+0985–U+0994 Independent vowels (অ আ ই ঈ উ ঊ ঋ ঌ এ ঐ ও ঔ) +U+0995–U+09B0 Consonants (ক–র) U+09B2 ল U+09B6–U+09B9 শ ষ স হ U+09BC Nukta @@ -195,23 +195,20 @@ BANGLA_STOP_WORDS = { Bengali has a defined sort order (স্বরবর্ণ before ব্যঞ্জনবর্ণ). Use locale-aware sorting: ```python -import locale - -# Set Bengali locale if available +# Preferred: use PyICU for reliable Bengali collation (pip install PyICU) try: - locale.setlocale(locale.LC_COLLATE, "bn_BD.UTF-8") - sorted_words = sorted(words, key=locale.strxfrm) -except locale.Error: - # Fallback: use ICU via PyICU import icu collator = icu.Collator.createInstance(icu.Locale("bn_BD")) sorted_words = sorted(words, key=collator.getSortKey) +except ImportError: + # Fallback: Unicode code point order (not linguistically perfect) + sorted_words = sorted(words) ``` ```javascript // JavaScript — Intl.Collator handles Bengali sort order const collator = new Intl.Collator('bn-BD'); -words.sort(collator.compare); +const sortedWords = [...words].sort(collator.compare); ``` ### Search with Normalization @@ -232,6 +229,11 @@ def bangla_search(query: str, corpus: list[str]) -> list[str]: ### Prompt Engineering for Bengali +**Important:** Bengali text, Banglish input, and web-scraped content should be +treated as untrusted user data. Always separate system instructions from +user-provided content with clear delimiters, and validate before executing +tool calls or destructive actions. + ```python # Use Bengali system prompts for Bengali-targeted apps system_prompt = ( @@ -245,6 +247,13 @@ system_prompt_mixed = ( "respond in Bengali. When they write in English, respond in English. " "For technical terms, you may use English words within Bengali sentences." ) + +# Always delimit user-provided content +def build_prompt(system: str, user_input: str) -> str: + return f"{system} +---USER INPUT--- +{user_input} +---END INPUT---" ``` ### Handling Banglish (Mixed Bengali-English) @@ -277,8 +286,8 @@ def prepare_bangla_dataset(texts: list[str]) -> list[str]: for text in texts: # Normalize Unicode text = unicodedata.normalize("NFC", text) - # Remove zero-width characters (common in web-scraped Bengali) - text = re.sub(r'[\u200b-\u200f\u2028-\u202f\ufeff]', '', text) + # Remove unwanted zero-width characters but preserve ZWNJ/ZWJ (important for conjuncts) + text = re.sub(r'[\u200b\u200e\u200f\u2028-\u202f\ufeff]', '', text) # Normalize whitespace text = re.sub(r'\s+', ' ', text).strip() if text: @@ -296,7 +305,10 @@ CREATE TABLE bengali_content ( body TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ); --- PostgreSQL: UTF-8 is the default, but set collation for sorting +-- PostgreSQL: UTF-8 is the default. Verify ICU collation availability first: +-- SELECT collname FROM pg_collation WHERE collname LIKE 'bn%'; +-- If missing, create it (requires ICU-enabled PostgreSQL build): +-- CREATE COLLATION IF NOT EXISTS "bn-BD-x-icu" (provider = icu, locale = 'bn-BD'); CREATE TABLE bengali_content ( id SERIAL PRIMARY KEY, title TEXT COLLATE "bn-BD-x-icu", @@ -338,7 +350,7 @@ if text.isdigit(): ... # GOOD — handles both import re -if re.match(r'^[\d০-৯]+$', text): ... +if re.fullmatch(r'[0-9০-৯]+', text): ... ``` ### Don't: Use ASCII Transliteration When Unicode Is Available @@ -357,6 +369,6 @@ title = "বাংলা ভাষা" # BAD — invisible characters cause matching failures text = scraped_html.get_text() -# GOOD — strip zero-width characters after extraction -text = re.sub(r'[\u200b-\u200f\u2028-\u202f\ufeff]', '', scraped_html.get_text()) +# GOOD — strip unwanted zero-width chars, preserving ZWNJ (U+200C) and ZWJ (U+200D) +text = re.sub(r'[\u200b\u200e\u200f\u2028-\u202f\ufeff]', '', scraped_html.get_text()) ```