📋 {title_text}
{pills_html}
{rows_html}
"""
async def _emit_todo_widget(
self,
chat_id: str,
lang: str,
emitter,
stats: Optional[Dict[str, Any]] = None,
force: bool = False,
) -> Dict[str, Any]:
"""Emit the TODO widget immediately when the snapshot actually changed."""
if not chat_id:
return {"emitted": False, "changed": False, "reason": "missing_chat_id"}
if not emitter:
return {"emitted": False, "changed": False, "reason": "missing_emitter"}
current_stats = (
stats
if stats is not None
else self._read_todo_status_from_session_db(chat_id)
)
snapshot_hash = self._compute_todo_widget_hash(current_stats)
previous_hash = self._read_todo_widget_hash(chat_id)
changed = force or snapshot_hash != previous_hash
if not changed:
return {
"emitted": False,
"changed": False,
"reason": "unchanged",
}
html_doc = self._build_todo_widget_html(lang, current_stats)
try:
await emitter({"type": "embeds", "data": {"embeds": [html_doc]}})
self._write_todo_widget_hash(chat_id, snapshot_hash)
return {
"emitted": True,
"changed": True,
"reason": "widget_updated",
}
except Exception as e:
logger.debug(f"[Todo Widget] Failed to emit widget: {e}")
return {"emitted": False, "changed": changed, "reason": str(e)}
def _query_mentions_todo_tables(self, query: str) -> bool:
"""Return whether a SQL query is operating on todo tables."""
if not query:
return False
return bool(re.search(r"\b(todos|todo_deps)\b", query, re.IGNORECASE))
def _get_shared_skills_dir(self, resolved_cwd: str) -> str:
"""Returns (and creates) the unified shared skills directory.
Both OpenWebUI page skills and pipe-installed skills live here.
The directory is persistent and shared across all sessions.
"""
shared_base = Path(self.valves.OPENWEBUI_SKILLS_SHARED_DIR or "").expanduser()
if not shared_base.is_absolute():
shared_base = Path(resolved_cwd) / shared_base
shared_dir = shared_base / "shared"
shared_dir.mkdir(parents=True, exist_ok=True)
return str(shared_dir)
def _parse_skill_md_meta(self, content: str, fallback_name: str) -> tuple:
"""Parse SKILL.md content into (name, description, body).
Handles files with or without YAML frontmatter.
Strips quotes from frontmatter string values.
"""
fm_match = re.match(r"^---\s*\n(.*?)\n---\s*\n", content, re.DOTALL)
if fm_match:
fm_text = fm_match.group(1)
body = content[fm_match.end() :].strip()
name = fallback_name
description = ""
for line in fm_text.split("\n"):
m = re.match(r"^name:\s*(.+)$", line)
if m:
name = m.group(1).strip().strip("\"'")
m = re.match(r"^description:\s*(.+)$", line)
if m:
description = m.group(1).strip().strip("\"'")
return name, description, body
# No frontmatter: try to extract H1 as name
h1_match = re.search(r"^#\s+(.+)$", content.strip(), re.MULTILINE)
name = h1_match.group(1).strip() if h1_match else fallback_name
return name, "", content.strip()
def _build_skill_md_content(self, name: str, description: str, body: str) -> str:
"""Construct a SKILL.md file string from name, description, and body."""
desc_line = description or name
if any(c in desc_line for c in ":#\n"):
desc_line = f'"{desc_line}"'
return (
f"---\n"
f"name: {name}\n"
f"description: {desc_line}\n"
f"---\n\n"
f"# {name}\n\n"
f"{body}\n"
)
def _sync_openwebui_skills(self, resolved_cwd: str, user_id: str) -> str:
"""Bidirectionally sync skills between OpenWebUI DB and the shared/ directory.
Sync rules (per skill):
DB → File: if a skill exists in OpenWebUI but has no directory entry, or the
DB is newer than the file → write/update SKILL.md in shared/.
File → DB: if a skill directory has no .owui_id or the file is newer than the
DB entry → create/update the skill in OpenWebUI DB.
Change detection uses MD5 content hash (skip if identical) then falls back to
timestamp comparison (db.updated_at vs file mtime) to determine direction.
A `.owui_id` marker file inside each skill directory tracks the OpenWebUI skill ID.
Skills installed via pipe that have no OpenWebUI counterpart are registered in DB.
If a directory has `.owui_id` but the corresponding OpenWebUI skill is gone,
the local directory is removed (UI is source of truth for deletions).
Returns the shared skills directory path (always, even on sync failure).
"""
shared_dir = Path(self._get_shared_skills_dir(resolved_cwd))
try:
from open_webui.models.skills import Skills, SkillForm, SkillMeta
sync_stats = {
"db_to_file_updates": 0,
"db_to_file_creates": 0,
"file_to_db_updates": 0,
"file_to_db_creates": 0,
"file_to_db_links": 0,
"orphan_dir_deletes": 0,
}
# ------------------------------------------------------------------
# Step 1: Load all accessible OpenWebUI skills
# ------------------------------------------------------------------
owui_by_id: Dict[str, dict] = {}
for skill in Skills.get_skills_by_user_id(user_id, "read") or []:
if not skill or not getattr(skill, "is_active", False):
continue
content = (getattr(skill, "content", "") or "").strip()
sk_id = str(getattr(skill, "id", "") or "")
sk_name = (getattr(skill, "name", "") or sk_id or "owui-skill").strip()
if not sk_id or not sk_name or not content:
continue
owui_by_id[sk_id] = {
"id": sk_id,
"name": sk_name,
"description": (getattr(skill, "description", "") or "")
.replace("\n", " ")
.strip(),
"content": content,
"updated_at": getattr(skill, "updated_at", 0) or 0,
}
# ------------------------------------------------------------------
# Step 2: Load directory skills (shared/) and build lookup maps
# ------------------------------------------------------------------
dir_skills: Dict[str, dict] = {} # dir_name → dict
for skill_dir in shared_dir.iterdir():
if not skill_dir.is_dir():
continue
skill_md_path = skill_dir / "SKILL.md"
if not skill_md_path.exists():
continue
owui_id_file = skill_dir / ".owui_id"
owui_id = (
owui_id_file.read_text(encoding="utf-8").strip()
if owui_id_file.exists()
else None
)
try:
file_content = skill_md_path.read_text(encoding="utf-8")
file_mtime = skill_md_path.stat().st_mtime
except Exception:
continue
dir_skills[skill_dir.name] = {
"path": skill_dir,
"owui_id": owui_id,
"mtime": file_mtime,
"content": file_content,
}
# Reverse map: owui_id → dir_name (for skills already linked)
id_to_dir: Dict[str, str] = {
info["owui_id"]: dn
for dn, info in dir_skills.items()
if info["owui_id"]
}
# ------------------------------------------------------------------
# Step 3: DB → File (OpenWebUI skills written to shared/)
# ------------------------------------------------------------------
for sk_id, sk in owui_by_id.items():
expected_file_content = self._build_skill_md_content(
sk["name"], sk["description"], sk["content"]
)
if sk_id in id_to_dir:
dir_name = id_to_dir[sk_id]
dir_info = dir_skills[dir_name]
existing_hash = hashlib.md5(
dir_info["content"].encode("utf-8", errors="replace")
).hexdigest()
new_hash = hashlib.md5(
expected_file_content.encode("utf-8", errors="replace")
).hexdigest()
if (
existing_hash != new_hash
and sk["updated_at"] > dir_info["mtime"]
):
# DB is newer — update file
(dir_info["path"] / "SKILL.md").write_text(
expected_file_content, encoding="utf-8"
)
dir_skills[dir_name]["content"] = expected_file_content
dir_skills[dir_name]["mtime"] = (
(dir_info["path"] / "SKILL.md").stat().st_mtime
)
sync_stats["db_to_file_updates"] += 1
else:
# No directory for this OpenWebUI skill → create one
dir_name = self._skill_dir_name_from_skill_name(sk["name"])
# Avoid collision with existing dir names
base = dir_name
suffix = 1
while dir_name in dir_skills:
dir_name = f"{base}-{suffix}"
suffix += 1
skill_dir = shared_dir / dir_name
skill_dir.mkdir(parents=True, exist_ok=True)
(skill_dir / "SKILL.md").write_text(
expected_file_content, encoding="utf-8"
)
(skill_dir / ".owui_id").write_text(sk_id, encoding="utf-8")
dir_skills[dir_name] = {
"path": skill_dir,
"owui_id": sk_id,
"mtime": (skill_dir / "SKILL.md").stat().st_mtime,
"content": expected_file_content,
}
id_to_dir[sk_id] = dir_name
sync_stats["db_to_file_creates"] += 1
# ------------------------------------------------------------------
# Step 4: File → DB (directory skills written to OpenWebUI)
# ------------------------------------------------------------------
owui_by_name: Dict[str, str] = {
info["name"]: sid for sid, info in owui_by_id.items()
}
for dir_name, dir_info in dir_skills.items():
owui_id = dir_info["owui_id"]
file_content = dir_info["content"]
file_mtime = dir_info["mtime"]
parsed_name, parsed_desc, parsed_body = self._parse_skill_md_meta(
file_content, dir_name
)
if owui_id and owui_id in owui_by_id:
# Skill is linked to DB — check if file is newer and content differs
db_info = owui_by_id[owui_id]
# Re-construct what the file would look like from DB to compare
db_file_content = self._build_skill_md_content(
db_info["name"], db_info["description"], db_info["content"]
)
file_hash = hashlib.md5(
file_content.encode("utf-8", errors="replace")
).hexdigest()
db_hash = hashlib.md5(
db_file_content.encode("utf-8", errors="replace")
).hexdigest()
if file_hash != db_hash and file_mtime > db_info["updated_at"]:
# File is newer — push to DB
Skills.update_skill_by_id(
owui_id,
{
"name": parsed_name,
"description": parsed_desc or parsed_name,
"content": parsed_body or file_content,
},
)
sync_stats["file_to_db_updates"] += 1
elif owui_id and owui_id not in owui_by_id:
# .owui_id points to a removed skill in OpenWebUI UI.
# UI is source of truth — delete local dir.
try:
shutil.rmtree(dir_info["path"], ignore_errors=False)
sync_stats["orphan_dir_deletes"] += 1
except Exception as e:
logger.warning(
f"[Skills Sync] Failed to remove orphaned skill dir '{dir_info['path']}': {e}"
)
else:
# No OpenWebUI link — try to match by name, then create new
matched_id = owui_by_name.get(parsed_name)
if matched_id:
# Link to existing skill with same name
(dir_info["path"] / ".owui_id").write_text(
matched_id, encoding="utf-8"
)
sync_stats["file_to_db_links"] += 1
db_info = owui_by_id[matched_id]
db_file_content = self._build_skill_md_content(
db_info["name"], db_info["description"], db_info["content"]
)
file_hash = hashlib.md5(
file_content.encode("utf-8", errors="replace")
).hexdigest()
db_hash = hashlib.md5(
db_file_content.encode("utf-8", errors="replace")
).hexdigest()
if file_hash != db_hash and file_mtime > db_info["updated_at"]:
Skills.update_skill_by_id(
matched_id,
{
"name": parsed_name,
"description": parsed_desc or parsed_name,
"content": parsed_body or file_content,
},
)
sync_stats["file_to_db_updates"] += 1
else:
# Truly new skill from file — register in OpenWebUI
new_skill = Skills.insert_new_skill(
user_id=user_id,
form_data=SkillForm(
id=str(uuid.uuid4()),
name=parsed_name,
description=parsed_desc or parsed_name,
content=parsed_body or file_content,
meta=SkillMeta(),
is_active=True,
),
)
if new_skill:
new_id = str(getattr(new_skill, "id", "") or "")
(dir_info["path"] / ".owui_id").write_text(
new_id, encoding="utf-8"
)
sync_stats["file_to_db_creates"] += 1
logger.debug(f"[Skills Sync] Summary: {sync_stats}")
except ImportError:
# Running outside OpenWebUI environment — directory is still usable
pass
except Exception as e:
logger.debug(f"[Copilot] Skills sync failed: {e}", exc_info=True)
return str(shared_dir)
def _resolve_session_skill_config(
self,
resolved_cwd: str,
user_id: str,
enable_openwebui_skills: bool,
disabled_skills: Optional[List[str]] = None,
) -> Dict[str, Any]:
skill_directories: List[str] = []
# Unified shared directory — always included.
# When enable_openwebui_skills is True, run bidirectional sync first so
# OpenWebUI page skills and directory skills are kept in sync.
if enable_openwebui_skills:
shared_dir = self._sync_openwebui_skills(resolved_cwd, user_id)
else:
shared_dir = self._get_shared_skills_dir(resolved_cwd)
skill_directories.append(shared_dir)
config: Dict[str, Any] = {}
if skill_directories:
config["skill_directories"] = self._dedupe_preserve_order(skill_directories)
if disabled_skills:
normalized_disabled = self._dedupe_preserve_order(disabled_skills)
if normalized_disabled:
config["disabled_skills"] = normalized_disabled
return config
def _is_code_interpreter_feature_enabled(
self, body: Optional[dict], __metadata__: Optional[dict] = None
) -> bool:
"""Code interpreter must be explicitly enabled by request feature flags."""
def _extract_flag(container: Any) -> Optional[bool]:
if not isinstance(container, dict):
return None
features = container.get("features")
if isinstance(features, dict) and "code_interpreter" in features:
return bool(features.get("code_interpreter"))
return None
# 1) top-level body.features
flag = _extract_flag(body)
if flag is not None:
return flag
# 2) body.metadata.features
if isinstance(body, dict):
flag = _extract_flag(body.get("metadata"))
if flag is not None:
return flag
# 3) injected __metadata__.features
flag = _extract_flag(__metadata__)
if flag is not None:
return flag
return False
async def _extract_system_prompt(
self,
body: dict,
messages: List[dict],
request_model: str,
real_model_id: str,
code_interpreter_enabled: bool = False,
__event_call__=None,
debug_enabled: bool = False,
) -> Tuple[Optional[str], str]:
"""Extract system prompt from metadata/model DB/body/messages."""
system_prompt_content: Optional[str] = None
system_prompt_source = ""
# 0) body.get("system_prompt") - Explicit Override (Highest Priority)
if hasattr(body, "get") and body.get("system_prompt"):
system_prompt_content = body.get("system_prompt")
system_prompt_source = "body_explicit_system_prompt"
await self._emit_debug_log(
f"Extracted system prompt from explicit body field (length: {len(system_prompt_content)})",
__event_call__,
debug_enabled=debug_enabled,
)
# 1) metadata.model.params.system
if not system_prompt_content:
metadata = body.get("metadata", {})
if isinstance(metadata, dict):
meta_model = metadata.get("model")
if isinstance(meta_model, dict):
meta_params = meta_model.get("params")
if isinstance(meta_params, dict) and meta_params.get("system"):
system_prompt_content = meta_params.get("system")
system_prompt_source = "metadata.model.params"
await self._emit_debug_log(
f"Extracted system prompt from metadata.model.params (length: {len(system_prompt_content)})",
__event_call__,
debug_enabled=debug_enabled,
)
# 2) model DB lookup
if not system_prompt_content:
try:
from open_webui.models.models import Models
model_ids_to_try = self._collect_model_ids(
body, request_model, real_model_id
)
await self._emit_debug_log(
f"Checking system prompt for models: {model_ids_to_try}",
__event_call__,
debug_enabled=debug_enabled,
)
for mid in model_ids_to_try:
model_record = Models.get_model_by_id(mid)
if model_record:
await self._emit_debug_log(
f"Checking Model DB for: {mid} (Record found: {model_record.id if hasattr(model_record, 'id') else 'Yes'})",
__event_call__,
debug_enabled=debug_enabled,
)
if hasattr(model_record, "params"):
params = model_record.params
if isinstance(params, dict):
system_prompt_content = params.get("system")
if system_prompt_content:
system_prompt_source = f"model_db:{mid}"
await self._emit_debug_log(
f"Success! Extracted system prompt from model DB using ID: {mid} (length: {len(system_prompt_content)})",
__event_call__,
debug_enabled=debug_enabled,
)
break
except Exception as e:
await self._emit_debug_log(
f"Failed to extract system prompt from model DB: {e}",
__event_call__,
debug_enabled=debug_enabled,
)
# 3) body.params.system
if not system_prompt_content:
body_params = body.get("params", {})
if isinstance(body_params, dict):
system_prompt_content = body_params.get("system")
if system_prompt_content:
system_prompt_source = "body_params"
await self._emit_debug_log(
f"Extracted system prompt from body.params.system (length: {len(system_prompt_content)})",
__event_call__,
debug_enabled=debug_enabled,
)
# 4) messages (role=system) - Last found wins or First found wins?
# Typically OpenWebUI puts the active system prompt as the FIRST message.
if not system_prompt_content:
for msg in messages:
if msg.get("role") == "system":
system_prompt_content = self._extract_text_from_content(
msg.get("content", "")
)
if system_prompt_content:
system_prompt_source = "messages_system"
await self._emit_debug_log(
f"Extracted system prompt from messages (reverse search) (length: {len(system_prompt_content)})",
__event_call__,
debug_enabled=debug_enabled,
)
break
# Append Code Interpreter Warning only when feature is explicitly enabled
if code_interpreter_enabled:
code_interpreter_warning = (
"\n\n[System Note]\n"
"The `execute_code` tool (builtin category: `code_interpreter`) executes code in a remote, ephemeral environment. "
"It cannot access files in your local workspace or persist changes. "
"Use it only for calculation or logic verification, not for file manipulation."
"\n"
"always use relative paths that start with `/api/v1/files/`. "
"Do not output `api/...` and do not prepend any domain or protocol (e.g., NEVER use `https://same.ai/api/...`)."
)
if system_prompt_content:
system_prompt_content += code_interpreter_warning
else:
system_prompt_content = code_interpreter_warning.strip()
return system_prompt_content, system_prompt_source
def _get_workspace_dir(self, user_id: str = None, chat_id: str = None) -> str:
"""Get the effective workspace directory with user and chat isolation."""
# Fixed base directory for OpenWebUI container
if os.path.exists("/app/backend/data"):
base_cwd = "/app/backend/data/copilot_workspace"
else:
# Local fallback for development environment
base_cwd = os.path.join(os.getcwd(), "copilot_workspace")
cwd = base_cwd
if user_id:
# Sanitize user_id to prevent path traversal
safe_user_id = re.sub(r"[^a-zA-Z0-9_-]", "_", str(user_id))
cwd = os.path.join(cwd, safe_user_id)
if chat_id:
# Sanitize chat_id
safe_chat_id = re.sub(r"[^a-zA-Z0-9_-]", "_", str(chat_id))
cwd = os.path.join(cwd, safe_chat_id)
# Ensure directory exists
if not os.path.exists(cwd):
try:
os.makedirs(cwd, exist_ok=True)
except Exception as e:
logger.error(f"Error creating workspace {cwd}: {e}")
return base_cwd
return cwd
def _build_client_config(self, user_id: str = None, chat_id: str = None) -> dict:
"""Build CopilotClient config from valves and request body."""
cwd = self._get_workspace_dir(user_id=user_id, chat_id=chat_id)
config_dir = self._get_copilot_config_dir()
# Set environment variable for SDK/CLI to pick up the new config location
os.environ["COPILOTSDK_CONFIG_DIR"] = config_dir
client_config = {}
if os.environ.get("COPILOT_CLI_PATH"):
client_config["cli_path"] = os.environ["COPILOT_CLI_PATH"]
client_config["cwd"] = cwd
client_config["config_dir"] = config_dir
if self.valves.LOG_LEVEL:
client_config["log_level"] = self.valves.LOG_LEVEL
if self.valves.LOG_LEVEL:
client_config["log_level"] = self.valves.LOG_LEVEL
# Setup persistent CLI tool installation directories
agent_env = dict(os.environ)
if os.path.exists("/app/backend/data"):
tools_dir = "/app/backend/data/.copilot_tools"
npm_dir = f"{tools_dir}/npm"
venv_dir = f"{tools_dir}/venv"
try:
os.makedirs(f"{npm_dir}/bin", exist_ok=True)
# Setup Python Virtual Environment to strictly protect system python
if not os.path.exists(f"{venv_dir}/bin/activate"):
import sys
subprocess.run(
[
sys.executable,
"-m",
"venv",
"--system-site-packages",
venv_dir,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=True,
)
agent_env["NPM_CONFIG_PREFIX"] = npm_dir
agent_env["VIRTUAL_ENV"] = venv_dir
agent_env.pop("PYTHONUSERBASE", None)
agent_env.pop("PIP_USER", None)
agent_env["PATH"] = (
f"{npm_dir}/bin:{venv_dir}/bin:{agent_env.get('PATH', '')}"
)
except Exception as e:
logger.warning(f"Failed to setup Python venv or tool dirs: {e}")
if self.valves.CUSTOM_ENV_VARS:
try:
custom_env = json.loads(self.valves.CUSTOM_ENV_VARS)
if isinstance(custom_env, dict):
agent_env.update(custom_env)
except:
pass
client_config["env"] = agent_env
return client_config
def _build_final_system_message(
self,
system_prompt_content: Optional[str],
is_admin: bool,
user_id: Optional[str],
chat_id: Optional[str],
manage_skills_intent: bool = False,
) -> str:
"""Build the final system prompt content used for both new and resumed sessions."""
try:
# -time.timezone is offset in seconds. UTC+8 is 28800.
is_china_tz = (-time.timezone / 3600) == 8.0
except Exception:
is_china_tz = False
if is_china_tz:
pkg_mirror_hint = " (Note: Server is in UTC+8. You MUST append `-i https://pypi.tuna.tsinghua.edu.cn/simple` for pip/uv and `--registry=https://registry.npmmirror.com` for npm to prevent network timeouts.)"
else:
pkg_mirror_hint = " (Note: If network is slow or times out, proactively use a fast regional mirror suitable for the current timezone.)"
system_parts = []
if system_prompt_content:
system_parts.append(system_prompt_content.strip())
if manage_skills_intent:
system_parts.append(
"[Skill Management]\n"
"If the user wants to install, create, delete, edit, or list skills, use the `manage_skills` tool.\n"
"Supported operations: list, install, create, edit, delete, show.\n"
"When installing skills that require CLI tools, you MAY run installation commands.\n"
f"To avoid hanging the session, ALWAYS append `-q` or `--silent` to package managers, and confirm unattended installations (e.g., `npm install -g -q