Files
multica/server/internal/handler/skill.go
Bohan Jiang 4233e82441 fix(skills): tree-first skill import to stop 504 on large repos (MUL-5136) (#5753)
Rework skills.sh and github.com skill imports around a single recursive
git-tree fetch to stop the 504 on large mono-repos (e.g. api-gateway-skill):

- One tree call replaces the per-directory contents crawl.
- Import caps checked arithmetically from tree metadata before any download
  (fail fast with 413 instead of timing out).
- Most-specific skill-dir resolution; repo root only as a last resort, which
  fixes the root SKILL.md name collision.
- Concurrent downloads (errgroup, limit 8).
- Overall 45s fetch deadline; cancellation is fatal on every supporting-file
  path (tree downloader, crawl listing/recursion/download, ClawHub) so a
  mid-download abort never persists a half-populated bundle.
- A skills.sh tree-fetch failure returns a retryable 503 instead of an unsafe
  root-directory fallback.
- Lenient conventional-path acceptance restored for both complete and truncated
  trees.
- maxImportFileCount 128 -> 256 (aligned daemon cap); 8 MiB bundle cap remains
  the real guard.
2026-07-22 14:38:31 +08:00

2639 lines
90 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package handler
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
skillpkg "github.com/multica-ai/multica/server/internal/skill"
db "github.com/multica-ai/multica/server/pkg/db/generated"
"github.com/multica-ai/multica/server/pkg/protocol"
"golang.org/x/sync/errgroup"
)
// sanitizeNullBytes makes a string safe for a PostgreSQL TEXT column.
//
// Two failure modes covered:
// - Embedded NUL (0x00) — PG rejects with SQLSTATE 22021. Removed.
// - Other invalid-UTF-8 byte sequences (e.g. 0x91 = Windows-1252 smart
// quote, which crashed agent-template import of skills containing
// Windows-encoded prose). `strings.ToValidUTF8` drops them.
//
// Name is kept for compatibility with the many call sites; the behaviour
// is a strict superset of the original.
func sanitizeNullBytes(s string) string {
return strings.ToValidUTF8(strings.ReplaceAll(s, "\x00", ""), "")
}
// --- Response structs ---
type SkillResponse struct {
ID string `json:"id"`
WorkspaceID string `json:"workspace_id"`
Name string `json:"name"`
Description string `json:"description"`
Content string `json:"content"`
Config any `json:"config"`
CreatedBy *string `json:"created_by"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
// SkillSummaryResponse is the list-endpoint shape: everything SkillResponse
// has except `content`. SKILL.md bodies routinely run 50200KB and shipping
// them in list payloads bloats responses past CLI timeouts on high-latency
// links (GH multica-ai/multica#2174). Detail endpoints still return the full
// SkillResponse with content.
type SkillSummaryResponse struct {
ID string `json:"id"`
WorkspaceID string `json:"workspace_id"`
Name string `json:"name"`
Description string `json:"description"`
Config any `json:"config"`
CreatedBy *string `json:"created_by"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
// Enabled is only populated for agent-scoped skill responses. Workspace
// skill lists describe the skill itself, so they omit assignment state.
Enabled *bool `json:"enabled,omitempty"`
}
// AgentSkillSummary is the still-narrower shape used for skills embedded in
// an Agent payload (`GET /api/agents`, `GET /api/agents/{id}`). The agent
// list batch query only joins enough columns to render the assignee chip in
// the UI; the standalone `/api/agents/{id}/skills` endpoint returns the full
// SkillSummaryResponse for callers that need the source/origin info.
type AgentSkillSummary struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Enabled bool `json:"enabled"`
}
type SkillFileResponse struct {
ID string `json:"id"`
SkillID string `json:"skill_id"`
Path string `json:"path"`
Content string `json:"content"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type SkillSearchCandidateResponse struct {
Name string `json:"name"`
URL string `json:"url"`
Source string `json:"source"`
Repo *string `json:"repo"`
InstallCount *int64 `json:"install_count"`
GitHubStars *int64 `json:"github_stars"`
Description string `json:"description"`
}
type SkillWithFilesResponse struct {
SkillResponse
Files []SkillFileResponse `json:"files"`
}
type SkillImportResult struct {
Status string `json:"status"`
Reason string `json:"reason,omitempty"`
Skill *SkillWithFilesResponse `json:"skill,omitempty"`
ExistingSkill *ExistingSkillIdentity `json:"existing_skill,omitempty"`
}
type ExistingSkillIdentity struct {
ID string `json:"id"`
Name string `json:"name"`
CreatedBy string `json:"created_by,omitempty"`
CanOverwrite bool `json:"can_overwrite,omitempty"`
}
func writeSkillImportDuplicateConflict(w http.ResponseWriter, existing ExistingSkillIdentity) {
writeJSON(w, http.StatusConflict, map[string]any{
"error": "a skill with this name already exists",
"existing_skill": existing,
})
}
func skillToResponse(s db.Skill) SkillResponse {
return SkillResponse{
ID: uuidToString(s.ID),
WorkspaceID: uuidToString(s.WorkspaceID),
Name: s.Name,
Description: s.Description,
Content: s.Content,
Config: decodeSkillConfig(s.Config),
CreatedBy: uuidToPtr(s.CreatedBy),
CreatedAt: timestampToString(s.CreatedAt),
UpdatedAt: timestampToString(s.UpdatedAt),
}
}
func (h *Handler) existingSkillIdentityByName(ctx context.Context, workspaceID pgtype.UUID, name string) (ExistingSkillIdentity, bool, error) {
skill, err := h.Queries.GetSkillByWorkspaceAndName(ctx, db.GetSkillByWorkspaceAndNameParams{
WorkspaceID: workspaceID,
Name: name,
})
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ExistingSkillIdentity{}, false, nil
}
return ExistingSkillIdentity{}, false, err
}
return existingSkillIdentity(skill, ""), true, nil
}
func existingSkillIdentity(skill db.Skill, userID string) ExistingSkillIdentity {
identity := ExistingSkillIdentity{
ID: uuidToString(skill.ID),
Name: skill.Name,
CanOverwrite: canOverwriteSkillByLocalImport(userID, skill),
}
if skill.CreatedBy.Valid {
identity.CreatedBy = uuidToString(skill.CreatedBy)
}
return identity
}
// decodeSkillConfig decodes a JSONB skill.config blob, defaulting to {} when
// missing or unparseable so the API surface always returns a JSON object.
func decodeSkillConfig(raw []byte) any {
var config any
if raw != nil {
_ = json.Unmarshal(raw, &config)
}
if config == nil {
return map[string]any{}
}
return config
}
func skillSummaryToResponse(
id, workspaceID pgtype.UUID,
name, description string,
config []byte,
createdBy pgtype.UUID,
createdAt, updatedAt pgtype.Timestamptz,
) SkillSummaryResponse {
return SkillSummaryResponse{
ID: uuidToString(id),
WorkspaceID: uuidToString(workspaceID),
Name: name,
Description: description,
Config: decodeSkillConfig(config),
CreatedBy: uuidToPtr(createdBy),
CreatedAt: timestampToString(createdAt),
UpdatedAt: timestampToString(updatedAt),
}
}
func skillFileToResponse(f db.SkillFile) SkillFileResponse {
return SkillFileResponse{
ID: uuidToString(f.ID),
SkillID: uuidToString(f.SkillID),
Path: f.Path,
Content: f.Content,
CreatedAt: timestampToString(f.CreatedAt),
UpdatedAt: timestampToString(f.UpdatedAt),
}
}
// --- Request structs ---
type CreateSkillRequest struct {
Name string `json:"name"`
Description string `json:"description"`
Content string `json:"content"`
Config any `json:"config"`
Files []CreateSkillFileRequest `json:"files,omitempty"`
}
type CreateSkillFileRequest struct {
Path string `json:"path"`
Content string `json:"content"`
}
type UpdateSkillRequest struct {
Name *string `json:"name"`
Description *string `json:"description"`
Content *string `json:"content"`
Config any `json:"config"`
Files []CreateSkillFileRequest `json:"files,omitempty"`
}
type SetAgentSkillsRequest struct {
SkillIDs []string `json:"skill_ids"`
}
type AddAgentSkillsRequest struct {
SkillIDs []string `json:"skill_ids"`
}
// --- Helpers ---
// validateFilePath checks that a file path is safe (no traversal, no absolute paths).
func validateFilePath(p string) bool {
if p == "" {
return false
}
if filepath.IsAbs(p) {
return false
}
cleaned := filepath.Clean(p)
if strings.HasPrefix(cleaned, "..") {
return false
}
return true
}
func (h *Handler) loadSkillForUser(w http.ResponseWriter, r *http.Request, id string) (db.Skill, bool) {
workspaceID := h.resolveWorkspaceID(r)
if workspaceID == "" {
writeError(w, http.StatusBadRequest, "workspace_id is required")
return db.Skill{}, false
}
skillUUID, ok := parseUUIDOrBadRequest(w, id, "skill id")
if !ok {
return db.Skill{}, false
}
skill, err := h.Queries.GetSkillInWorkspace(r.Context(), db.GetSkillInWorkspaceParams{
ID: skillUUID,
WorkspaceID: parseUUID(workspaceID),
})
if err != nil {
writeError(w, http.StatusNotFound, "skill not found")
return skill, false
}
return skill, true
}
// --- Skill CRUD ---
func (h *Handler) ListSkills(w http.ResponseWriter, r *http.Request) {
workspaceID := h.resolveWorkspaceID(r)
skills, err := h.Queries.ListSkillSummariesByWorkspace(r.Context(), parseUUID(workspaceID))
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to list skills")
return
}
resp := make([]SkillSummaryResponse, len(skills))
for i, s := range skills {
resp[i] = skillSummaryToResponse(
s.ID, s.WorkspaceID, s.Name, s.Description, s.Config,
s.CreatedBy, s.CreatedAt, s.UpdatedAt,
)
}
writeJSON(w, http.StatusOK, resp)
}
func (h *Handler) SearchSkills(w http.ResponseWriter, r *http.Request) {
query := strings.TrimSpace(r.URL.Query().Get("q"))
if query == "" {
writeError(w, http.StatusBadRequest, "query is required")
return
}
httpClient := &http.Client{Timeout: 30 * time.Second}
candidates, err := searchClawHubSkills(httpClient, query)
if err != nil {
writeJSON(w, http.StatusBadGateway, map[string]string{
"code": "upstream_unavailable",
"error": err.Error(),
})
return
}
writeJSON(w, http.StatusOK, candidates)
}
func (h *Handler) GetSkill(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
skill, ok := h.loadSkillForUser(w, r, id)
if !ok {
return
}
files, err := h.Queries.ListSkillFiles(r.Context(), skill.ID)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to list skill files")
return
}
fileResps := make([]SkillFileResponse, len(files))
for i, f := range files {
fileResps[i] = skillFileToResponse(f)
}
writeJSON(w, http.StatusOK, SkillWithFilesResponse{
SkillResponse: skillToResponse(skill),
Files: fileResps,
})
}
func (h *Handler) CreateSkill(w http.ResponseWriter, r *http.Request) {
workspaceID := h.resolveWorkspaceID(r)
creatorID, ok := requireUserID(w, r)
if !ok {
return
}
workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id")
if !ok {
return
}
creatorUUID := parseUUID(creatorID)
var req CreateSkillRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Name == "" {
writeError(w, http.StatusBadRequest, "name is required")
return
}
for _, f := range req.Files {
if !validateFilePath(f.Path) {
writeError(w, http.StatusBadRequest, "invalid file path: "+f.Path)
return
}
}
resp, err := h.createSkillWithFiles(r.Context(), skillCreateInput{
WorkspaceID: workspaceUUID,
CreatorID: creatorUUID,
Name: req.Name,
Description: req.Description,
Content: req.Content,
Config: req.Config,
Files: req.Files,
})
if err != nil {
if isUniqueViolation(err) {
writeError(w, http.StatusConflict, "a skill with this name already exists")
return
}
writeError(w, http.StatusInternalServerError, "failed to create skill: "+err.Error())
return
}
actorType, actorID := h.resolveActor(r, creatorID, workspaceID)
h.publish(protocol.EventSkillCreated, workspaceID, actorType, actorID, map[string]any{"skill": resp})
writeJSON(w, http.StatusCreated, resp)
}
// canManageSkill checks whether the current user can update or delete a skill.
// The skill creator or workspace owner/admin can manage any skill.
func (h *Handler) canManageSkill(w http.ResponseWriter, r *http.Request, skill db.Skill) bool {
wsID := uuidToString(skill.WorkspaceID)
member, ok := h.requireWorkspaceRole(w, r, wsID, "skill not found", "owner", "admin", "member")
if !ok {
return false
}
isAdmin := roleAllowed(member.Role, "owner", "admin")
isSkillCreator := skill.CreatedBy.Valid && uuidToString(skill.CreatedBy) == requestUserID(r)
if !isAdmin && !isSkillCreator {
writeError(w, http.StatusForbidden, "only the skill creator can manage this skill")
return false
}
return true
}
// canOverwriteSkillByLocalImport reports whether userID may overwrite skill via
// a runtime-local-skill re-import. This is intentionally NARROWER than
// canManageSkill: only the original creator may overwrite by re-importing.
// Workspace owners/admins who want to change a skill they did not create must
// edit it in-app instead. See MUL-2701 / MUL-2800.
func canOverwriteSkillByLocalImport(userID string, skill db.Skill) bool {
return skill.CreatedBy.Valid && uuidToString(skill.CreatedBy) == userID
}
func (h *Handler) UpdateSkill(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
skill, ok := h.loadSkillForUser(w, r, id)
if !ok {
return
}
if !h.canManageSkill(w, r, skill) {
return
}
var req UpdateSkillRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
for _, f := range req.Files {
if !validateFilePath(f.Path) {
writeError(w, http.StatusBadRequest, "invalid file path: "+f.Path)
return
}
}
tx, err := h.TxStarter.Begin(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to start transaction")
return
}
defer tx.Rollback(r.Context())
qtx := h.Queries.WithTx(tx)
params := db.UpdateSkillParams{
ID: parseUUID(id),
}
if req.Name != nil {
params.Name = pgtype.Text{String: sanitizeNullBytes(*req.Name), Valid: true}
}
if req.Description != nil {
params.Description = pgtype.Text{String: sanitizeNullBytes(*req.Description), Valid: true}
}
if req.Content != nil {
params.Content = pgtype.Text{String: sanitizeNullBytes(*req.Content), Valid: true}
}
if req.Config != nil {
config, _ := json.Marshal(req.Config)
params.Config = config
}
skill, err = qtx.UpdateSkill(r.Context(), params)
if err != nil {
if isUniqueViolation(err) {
writeError(w, http.StatusConflict, "a skill with this name already exists")
return
}
writeError(w, http.StatusInternalServerError, "failed to update skill: "+err.Error())
return
}
// If files are provided, replace all files.
var fileResps []SkillFileResponse
if req.Files != nil {
if err := qtx.DeleteSkillFilesBySkill(r.Context(), skill.ID); err != nil {
writeError(w, http.StatusInternalServerError, "failed to delete old skill files")
return
}
fileResps = make([]SkillFileResponse, 0, len(req.Files))
for _, f := range req.Files {
// SKILL.md is reserved for the primary skill content (skill.Content).
if skillpkg.IsReservedContentPath(f.Path) {
continue
}
sf, err := qtx.UpsertSkillFile(r.Context(), db.UpsertSkillFileParams{
SkillID: skill.ID,
Path: sanitizeNullBytes(f.Path),
Content: sanitizeNullBytes(f.Content),
})
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to upsert skill file: "+err.Error())
return
}
fileResps = append(fileResps, skillFileToResponse(sf))
}
} else {
files, _ := qtx.ListSkillFiles(r.Context(), skill.ID)
fileResps = make([]SkillFileResponse, len(files))
for i, f := range files {
fileResps[i] = skillFileToResponse(f)
}
}
if err := tx.Commit(r.Context()); err != nil {
writeError(w, http.StatusInternalServerError, "failed to commit")
return
}
resp := SkillWithFilesResponse{
SkillResponse: skillToResponse(skill),
Files: fileResps,
}
wsID := h.resolveWorkspaceID(r)
actorType, actorID := h.resolveActor(r, requestUserID(r), wsID)
h.publish(protocol.EventSkillUpdated, wsID, actorType, actorID, map[string]any{"skill": resp})
writeJSON(w, http.StatusOK, resp)
}
func (h *Handler) DeleteSkill(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
skill, ok := h.loadSkillForUser(w, r, id)
if !ok {
return
}
if !h.canManageSkill(w, r, skill) {
return
}
tx, err := h.TxStarter.Begin(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to start transaction")
return
}
defer tx.Rollback(r.Context())
qtx := h.Queries.WithTx(tx)
if err := qtx.DeleteSkillLabelAssignmentsBySkill(r.Context(), skill.ID); err != nil {
writeError(w, http.StatusInternalServerError, "failed to remove skill label assignments")
return
}
if err := qtx.DeleteSkill(r.Context(), db.DeleteSkillParams{
ID: skill.ID,
WorkspaceID: skill.WorkspaceID,
}); err != nil {
writeError(w, http.StatusInternalServerError, "failed to delete skill")
return
}
if err := tx.Commit(r.Context()); err != nil {
writeError(w, http.StatusInternalServerError, "failed to commit skill deletion")
return
}
actorType, actorID := h.resolveActor(r, requestUserID(r), uuidToString(skill.WorkspaceID))
h.publish(protocol.EventSkillDeleted, uuidToString(skill.WorkspaceID), actorType, actorID, map[string]any{"skill_id": uuidToString(skill.ID)})
w.WriteHeader(http.StatusNoContent)
}
// --- Skill import ---
type ImportSkillRequest struct {
URL string `json:"url"`
OnConflict string `json:"on_conflict,omitempty"`
}
const (
importOnConflictFail = "fail"
importOnConflictOverwrite = "overwrite"
importOnConflictRename = "rename"
importOnConflictSkip = "skip"
)
const maxImportRenameAttempts = 50
func validImportOnConflict(strategy string) bool {
switch strategy {
case "", importOnConflictFail, importOnConflictOverwrite, importOnConflictRename, importOnConflictSkip:
return true
}
return false
}
// Per-import bundle limits. These mirror the local-runtime importer so that
// URL imports cannot smuggle in payloads that the rest of the stack would
// reject. fetchRawFile enforces the per-file cap; importedSkill.addFile
// enforces the bundle-wide caps.
const (
maxImportFileSize = 1 << 20 // 1 MiB per file
maxImportTotalSize = 8 << 20 // 8 MiB per import bundle (sum of supporting files)
maxImportFileCount = 256 // max number of supporting files
)
// importedSkill holds the data extracted from an external source.
type importedSkill struct {
name string
description string
content string // SKILL.md body
files []importedFile
bundleSize int // running sum of file content bytes for cap enforcement
origin map[string]any // written into skill.config.origin so the UI can show provenance
}
type importedFile struct {
path string
content string
}
// errImportCapExceeded marks an error caused by a per-file or per-bundle cap.
// Such errors must abort the import — silently dropping a file would otherwise
// produce an incomplete skill that looks valid to the user.
var errImportCapExceeded = errors.New("import cap exceeded")
// errImportSourceUnavailable marks a transient failure to read the upstream
// source (e.g. the GitHub tree API rate limiting). The import can't proceed
// safely but should be retried, so it maps to a retryable HTTP status rather
// than a permanent error.
var errImportSourceUnavailable = errors.New("import source temporarily unavailable")
// isCapError reports whether err is (or wraps) errImportCapExceeded.
func isCapError(err error) bool {
return errors.Is(err, errImportCapExceeded)
}
// addFile appends a supporting file while enforcing the per-bundle caps. It
// returns an error when either the file count or aggregate byte budget would
// be exceeded so the caller fails the import instead of silently truncating.
//
// Binary files (images, fonts, archives) are silently skipped: their bytes
// can't survive a PG TEXT column (SQLSTATE 22021), and they're reference
// assets the agent never reads as text anyway. Logging the skip leaves a
// breadcrumb if a user expected one of these to import.
func (s *importedSkill) addFile(path, content string) error {
if isLikelyBinaryFilePath(path) {
slog.Info("skill import: skipping binary file", "path", path, "size", len(content))
return nil
}
if len(s.files) >= maxImportFileCount {
return fmt.Errorf("%w: import bundle exceeds %d file limit", errImportCapExceeded, maxImportFileCount)
}
if s.bundleSize+len(content) > maxImportTotalSize {
return fmt.Errorf("%w: import bundle exceeds %d byte limit", errImportCapExceeded, maxImportTotalSize)
}
s.bundleSize += len(content)
s.files = append(s.files, importedFile{path: path, content: content})
return nil
}
// isLikelyBinaryFilePath reports whether the file's extension indicates a
// non-text payload. Conservative blacklist — extensions not on the list
// are assumed text and pass through. `sanitizeNullBytes` (called at PG
// insert time) is the second-line defence against any text file that
// turns out to have stray invalid-UTF-8 bytes.
func isLikelyBinaryFilePath(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
switch ext {
case
// images
".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".ico", ".heic",
// fonts
".ttf", ".otf", ".woff", ".woff2", ".eot",
// archives
".zip", ".gz", ".tar", ".bz2", ".7z", ".rar",
// documents (binary office)
".pdf", ".docx", ".xlsx", ".pptx", ".doc", ".xls", ".ppt",
// media
".mp3", ".mp4", ".wav", ".avi", ".mov", ".webm", ".m4a", ".flac",
// compiled / executable
".exe", ".dll", ".so", ".dylib", ".class", ".jar", ".wasm",
// db / cache
".db", ".sqlite", ".sqlite3", ".pyc":
return true
}
return false
}
// --- ClawHub types ---
var clawHubAPIBase = "https://clawhub.ai/api/v1"
const clawHubSearchStatsLimit = 10
type clawhubSearchResponse struct {
Results []clawhubSearchResult `json:"results"`
}
type clawhubSearchResult struct {
Slug string `json:"slug"`
DisplayName string `json:"displayName"`
Summary string `json:"summary"`
OwnerHandle string `json:"ownerHandle"`
}
type clawhubSkillStats struct {
InstallsAllTime int64 `json:"installsAllTime"`
InstallsCurrent int64 `json:"installsCurrent"`
}
type clawhubGetSkillResponse struct {
Skill clawhubSkill `json:"skill"`
LatestVersion *clawhubLatestVersion `json:"latestVersion"`
}
type clawhubSkill struct {
Slug string `json:"slug"`
DisplayName string `json:"displayName"`
Summary string `json:"summary"`
Tags map[string]string `json:"tags"`
Stats clawhubSkillStats `json:"stats"`
}
type clawhubLatestVersion struct {
Version string `json:"version"`
}
type clawhubVersionDetailResponse struct {
Version clawhubVersionDetail `json:"version"`
}
type clawhubVersionDetail struct {
Version string `json:"version"`
Files []clawhubFileEntry `json:"files"`
}
type clawhubFileEntry struct {
Path string `json:"path"`
Size int64 `json:"size"`
}
// --- GitHub types (for skills.sh) ---
type githubContentEntry struct {
Name string `json:"name"`
Path string `json:"path"`
Type string `json:"type"` // "file" or "dir"
URL string `json:"url"`
DownloadURL string `json:"download_url"`
}
type githubRepoInfo struct {
DefaultBranch string `json:"default_branch"`
}
type githubTreeResponse struct {
Tree []githubTreeEntry `json:"tree"`
Truncated bool `json:"truncated"`
}
type githubTreeEntry struct {
Path string `json:"path"`
Type string `json:"type"` // "blob" or "tree"
Size int64 `json:"size"` // blob byte size (absent/0 for tree entries)
}
// fetchGitHubDefaultBranch returns the default branch of a GitHub repository.
// Falls back to "main" if the API call fails.
func fetchGitHubDefaultBranch(ctx context.Context, httpClient *http.Client, owner, repo string) string {
apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s",
url.PathEscape(owner), url.PathEscape(repo))
resp, err := doGitHubAPIGet(ctx, httpClient, apiURL)
if err != nil || resp.StatusCode != http.StatusOK {
if resp != nil {
resp.Body.Close()
}
return "main"
}
defer resp.Body.Close()
var info githubRepoInfo
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil || info.DefaultBranch == "" {
return "main"
}
return info.DefaultBranch
}
// --- URL detection ---
// importSource identifies where a URL points.
type importSource int
const (
sourceClawHub importSource = iota
sourceSkillsSh
sourceGitHub
)
// detectImportSource determines the source from a URL.
// Returns the source and a normalized URL (with scheme).
func detectImportSource(raw string) (importSource, string, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0, "", fmt.Errorf("empty URL")
}
normalized := raw
if !strings.HasPrefix(normalized, "http://") && !strings.HasPrefix(normalized, "https://") {
normalized = "https://" + normalized
}
parsed, err := url.Parse(normalized)
if err != nil {
return 0, "", fmt.Errorf("invalid URL: %w", err)
}
host := strings.ToLower(parsed.Hostname())
switch {
case host == "skills.sh" || host == "www.skills.sh":
return sourceSkillsSh, normalized, nil
case host == "clawhub.ai" || host == "www.clawhub.ai":
return sourceClawHub, normalized, nil
case host == "github.com" || host == "www.github.com":
return sourceGitHub, normalized, nil
default:
// If no host (bare slug), default to clawhub
if !strings.Contains(raw, "/") || !strings.Contains(raw, ".") {
return sourceClawHub, raw, nil
}
return 0, "", fmt.Errorf("unsupported source: %s (supported: clawhub.ai, skills.sh, github.com)", host)
}
}
// --- ClawHub import ---
// parseClawHubSlug extracts the skill slug from a clawhub.ai URL.
func parseClawHubSlug(raw string) (string, error) {
parsed, err := url.Parse(raw)
if err != nil {
return "", fmt.Errorf("invalid URL: %w", err)
}
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
// /{owner}/{slug} — take the last segment as the slug
if len(parts) == 2 {
return parts[1], nil
}
if len(parts) == 1 && parts[0] != "" {
return parts[0], nil
}
// Bare slug (no path)
if raw == parsed.Host || parsed.Path == "" || parsed.Path == "/" {
return "", fmt.Errorf("missing skill slug in URL")
}
return "", fmt.Errorf("could not extract skill slug from URL: %s", raw)
}
func searchClawHubSkills(httpClient *http.Client, query string) ([]SkillSearchCandidateResponse, error) {
searchURL := clawHubAPIBase + "/search?q=" + url.QueryEscape(query)
resp, err := httpClient.Get(searchURL)
if err != nil {
return nil, fmt.Errorf("failed to reach ClawHub: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("ClawHub search returned status %d", resp.StatusCode)
}
var searchResp clawhubSearchResponse
if err := json.NewDecoder(resp.Body).Decode(&searchResp); err != nil {
return nil, fmt.Errorf("failed to parse ClawHub search response")
}
candidates := make([]SkillSearchCandidateResponse, 0, len(searchResp.Results))
for i, result := range searchResp.Results {
if result.Slug == "" {
continue
}
candidate := SkillSearchCandidateResponse{
Name: result.DisplayName,
URL: buildClawHubSkillURL(result.OwnerHandle, result.Slug),
Source: "clawhub.ai",
Description: result.Summary,
}
if candidate.Name == "" {
candidate.Name = result.Slug
}
if i < clawHubSearchStatsLimit {
if count, ok := fetchClawHubInstallCount(httpClient, result.Slug); ok {
candidate.InstallCount = &count
}
}
candidates = append(candidates, candidate)
}
return candidates, nil
}
func buildClawHubSkillURL(ownerHandle, slug string) string {
if ownerHandle == "" {
return "https://clawhub.ai/" + url.PathEscape(slug)
}
return "https://clawhub.ai/" + url.PathEscape(ownerHandle) + "/" + url.PathEscape(slug)
}
func fetchClawHubInstallCount(httpClient *http.Client, slug string) (int64, bool) {
detailURL := clawHubAPIBase + "/skills/" + url.PathEscape(slug)
resp, err := httpClient.Get(detailURL)
if err != nil {
slog.Warn("clawhub search: failed to fetch skill details", "slug", slug, "error", err)
return 0, false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
slog.Warn("clawhub search: skill details returned non-200", "slug", slug, "status", resp.StatusCode)
return 0, false
}
var detail clawhubGetSkillResponse
if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil {
slog.Warn("clawhub search: failed to parse skill details", "slug", slug, "error", err)
return 0, false
}
if detail.Skill.Stats.InstallsAllTime > 0 {
return detail.Skill.Stats.InstallsAllTime, true
}
return detail.Skill.Stats.InstallsCurrent, true
}
func fetchFromClawHub(ctx context.Context, httpClient *http.Client, rawURL string) (*importedSkill, error) {
slug, err := parseClawHubSlug(rawURL)
if err != nil {
return nil, err
}
apiBase := clawHubAPIBase
// 1. Fetch skill metadata
skillReq, err := http.NewRequestWithContext(ctx, http.MethodGet, apiBase+"/skills/"+url.PathEscape(slug), nil)
if err != nil {
return nil, err
}
skillResp, err := httpClient.Do(skillReq)
if err != nil {
return nil, fmt.Errorf("failed to reach ClawHub: %w", err)
}
defer skillResp.Body.Close()
if skillResp.StatusCode == http.StatusNotFound {
return nil, fmt.Errorf("skill not found on ClawHub: %s", slug)
}
if skillResp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("ClawHub returned status %d", skillResp.StatusCode)
}
var chResp clawhubGetSkillResponse
if err := json.NewDecoder(skillResp.Body).Decode(&chResp); err != nil {
return nil, fmt.Errorf("failed to parse ClawHub response")
}
chSkill := chResp.Skill
// 2. Determine latest version and fetch file list
latestVersion := ""
if v, ok := chSkill.Tags["latest"]; ok {
latestVersion = v
} else if chResp.LatestVersion != nil {
latestVersion = chResp.LatestVersion.Version
}
var filePaths []string
if latestVersion != "" {
vURL := fmt.Sprintf("%s/skills/%s/versions/%s", apiBase, url.PathEscape(slug), url.PathEscape(latestVersion))
vReq, verr := http.NewRequestWithContext(ctx, http.MethodGet, vURL, nil)
if verr != nil {
return nil, verr
}
vResp, err := httpClient.Do(vReq)
if err == nil {
defer vResp.Body.Close()
if vResp.StatusCode == http.StatusOK {
var vDetail clawhubVersionDetailResponse
if err := json.NewDecoder(vResp.Body).Decode(&vDetail); err == nil {
for _, f := range vDetail.Version.Files {
filePaths = append(filePaths, f.Path)
}
}
}
}
}
// 3. Download each file
result := &importedSkill{
name: chSkill.DisplayName,
description: chSkill.Summary,
origin: map[string]any{
"type": "clawhub",
"source_url": rawURL,
"slug": slug,
},
}
if result.name == "" {
result.name = slug
}
for _, fp := range filePaths {
fileURL := fmt.Sprintf("%s/skills/%s/file?path=%s", apiBase, url.PathEscape(slug), url.QueryEscape(fp))
if latestVersion != "" {
fileURL += "&version=" + url.QueryEscape(latestVersion)
}
body, err := fetchRawFile(ctx, httpClient, fileURL)
if err != nil {
// Cap violations must abort: silently dropping a file would
// produce an incomplete bundle that looks valid. SKILL.md is
// load-bearing, so any failure on it is fatal too.
if isCapError(err) || fp == "SKILL.md" {
return nil, fmt.Errorf("clawhub import: %s: %w", fp, err)
}
// A cancelled context (overall deadline / client disconnect) is
// fatal for the same reason: skipping every remaining file would
// persist a half-populated bundle as a success.
if ctx.Err() != nil {
return nil, fmt.Errorf("clawhub import: fetch aborted at %s: %w", fp, ctx.Err())
}
slog.Warn("clawhub import: file download failed", "path", fp, "error", err)
continue
}
if fp == "SKILL.md" {
result.content = string(body)
continue
}
if err := result.addFile(fp, string(body)); err != nil {
return nil, err
}
}
if result.content == "" {
return nil, fmt.Errorf("clawhub import: SKILL.md is empty or missing for %s", slug)
}
return result, nil
}
// --- skills.sh import ---
// parseSkillsShParts extracts owner, repo, skill-name from a skills.sh URL.
// URL format: https://skills.sh/{owner}/{repo}/{skill-name}
func parseSkillsShParts(raw string) (owner, repo, skillName string, err error) {
parsed, err := url.Parse(raw)
if err != nil {
return "", "", "", fmt.Errorf("invalid URL: %w", err)
}
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
if len(parts) != 3 {
return "", "", "", fmt.Errorf("expected URL format: skills.sh/{owner}/{repo}/{skill-name}, got: %s", parsed.Path)
}
return parts[0], parts[1], parts[2], nil
}
func fetchFromSkillsSh(ctx context.Context, httpClient *http.Client, rawURL string) (*importedSkill, error) {
owner, repo, skillName, err := parseSkillsShParts(rawURL)
if err != nil {
return nil, err
}
// A skills.sh URL maps onto a GitHub repository. Both skill-directory
// resolution and supporting-file enumeration are driven by a single
// recursive tree fetch. This collapses what used to be one contents-API
// call per directory — hundreds of sequential requests for a large mono-repo
// like api-gateway-skill, which pushed the request past the reverse-proxy
// gateway timeout (504) — into one call, and lets the import caps be checked
// from the tree metadata before any file is downloaded.
defaultBranch := fetchGitHubDefaultBranch(ctx, httpClient, owner, repo)
rawPrefix := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s",
url.PathEscape(owner), url.PathEscape(repo), url.PathEscape(defaultBranch))
tree, truncated, treeErr := fetchGitHubTree(ctx, httpClient, owner, repo, defaultBranch)
if treeErr != nil {
// The tree fetch failed (typically GitHub API rate limiting, which also
// takes down the contents API). Without the tree we cannot safely
// resolve which directory is the skill: a raw-probe + root-SKILL.md
// fallback would re-select the repository root whenever its name
// collides with the slug (the api-gateway-skill case) and the same
// rate limiting would then leave the crawl empty — persisting the wrong
// SKILL.md with zero supporting files as a "successful" import. Fail
// with a retryable error instead so nothing incorrect is saved.
slog.Warn("skills.sh import: repository tree fetch failed",
"owner", owner, "repo", repo, "error", treeErr)
return nil, fmt.Errorf("%w: could not read the %s/%s repository tree (usually GitHub API rate limiting — set GITHUB_TOKEN on the server or retry): %v",
errImportSourceUnavailable, owner, repo, treeErr)
}
skillDir, skillMdBody, err := resolveSkillDirFromTree(ctx, httpClient, owner, repo, defaultBranch, rawPrefix, skillName, tree, truncated)
if err != nil {
return nil, err
}
result := newSkillsShImportedSkill(skillMdBody, skillName, rawURL, owner, repo)
if truncated {
// The tree is incomplete, so it can't drive enumeration; use the legacy
// per-directory crawl scoped to the resolved skill directory.
if err := addSupportingFilesViaCrawl(ctx, httpClient, result, owner, repo, defaultBranch, skillDir); err != nil {
return nil, err
}
return result, nil
}
if err := addSupportingFilesFromTree(ctx, httpClient, result, tree, rawPrefix, skillDir); err != nil {
return nil, err
}
return result, nil
}
// newSkillsShImportedSkill builds the importedSkill shell (name, description,
// SKILL.md content, provenance) from a fetched SKILL.md body.
func newSkillsShImportedSkill(skillMdBody []byte, skillName, rawURL, owner, repo string) *importedSkill {
name, description := skillpkg.ParseSkillFrontmatter(string(skillMdBody))
if name == "" {
name = skillName
}
return &importedSkill{
name: name,
description: description,
content: string(skillMdBody),
origin: map[string]any{
"type": "skills_sh",
"source_url": rawURL,
"owner": owner,
"repo": repo,
"skill": skillName,
},
}
}
// addSupportingFilesViaCrawl is the legacy per-directory enumeration path, used
// as a fallback only for a truncated recursive tree (and by the github.com
// importer). A tree-fetch failure no longer routes here — it returns a retryable
// error instead. It lists skillDir via the contents API, recurses
// subdirectories, and downloads each file. It stays lenient on a genuine listing
// failure (returns with only SKILL.md, matching prior behavior under GitHub API
// rate limiting) but treats a cancelled context as fatal so a mid-crawl deadline
// or disconnect can't persist an incomplete bundle as a success.
func addSupportingFilesViaCrawl(ctx context.Context, httpClient *http.Client, result *importedSkill, owner, repo, ref, skillDir string) error {
apiURL := buildGitHubContentsURL(owner, repo, skillDir, ref)
dirResp, err := doGitHubAPIGet(ctx, httpClient, apiURL)
if err != nil || dirResp.StatusCode != http.StatusOK {
if dirResp != nil {
dirResp.Body.Close()
}
// A cancelled context (overall deadline / client disconnect) must abort
// rather than being swallowed as "no supporting files", which would
// persist an incomplete bundle as a success.
if ctx.Err() != nil {
return fmt.Errorf("github import: directory listing aborted: %w", ctx.Err())
}
return nil
}
defer dirResp.Body.Close()
var entries []githubContentEntry
if err := json.NewDecoder(dirResp.Body).Decode(&entries); err != nil {
// A cancelled context surfaces as a decode error when the deadline or a
// client disconnect lands while the response body is still being read.
// Treat it as fatal rather than swallowing it — otherwise the import is
// saved with a valid SKILL.md but zero supporting files.
if ctx.Err() != nil {
return fmt.Errorf("github import: directory listing read aborted: %w", ctx.Err())
}
slog.Warn("github import: failed to decode top-level directory listing", "url", apiURL, "error", err)
return nil
}
var allFiles []githubContentEntry
collectGitHubFiles(ctx, httpClient, entries, &allFiles, apiURL)
// collectGitHubFiles is lenient on a failed subdirectory listing; if the
// context was cancelled mid-crawl the collected set is incomplete, so abort.
if ctx.Err() != nil {
return fmt.Errorf("github import: directory crawl aborted: %w", ctx.Err())
}
basePath := ""
if skillDir != "" {
basePath = skillDir + "/"
}
for _, entry := range allFiles {
if entry.DownloadURL == "" {
continue
}
body, err := fetchRawFile(ctx, httpClient, entry.DownloadURL)
if err != nil {
if isCapError(err) {
return fmt.Errorf("github import: %s: %w", entry.Path, err)
}
if ctx.Err() != nil {
return fmt.Errorf("github import: fetch aborted at %s: %w", entry.Path, ctx.Err())
}
slog.Warn("github import: file download failed", "path", entry.Path, "error", err)
continue
}
relPath := strings.TrimPrefix(entry.Path, basePath)
if err := result.addFile(relPath, string(body)); err != nil {
return err
}
}
return nil
}
// fetchGitHubTree fetches the full recursive git tree for a ref in a single API
// call. Each returned blob entry carries its path and byte size, which lets the
// importer both resolve the skill directory and enforce the import caps without
// the per-directory contents crawl that previously issued one API call per
// directory — hundreds of sequential requests for a large mono-repo, which is
// what pushed skills.sh imports past the reverse-proxy gateway timeout (504).
func fetchGitHubTree(ctx context.Context, httpClient *http.Client, owner, repo, ref string) (entries []githubTreeEntry, truncated bool, err error) {
apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees/%s?recursive=1",
url.PathEscape(owner), url.PathEscape(repo), url.PathEscape(ref))
resp, err := doGitHubAPIGet(ctx, httpClient, apiURL)
if err != nil {
return nil, false, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, false, fmt.Errorf("HTTP %d", resp.StatusCode)
}
var tree githubTreeResponse
if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil {
return nil, false, err
}
return tree.Tree, tree.Truncated, nil
}
// resolveSkillDirFromTree resolves the skill directory and its SKILL.md body
// from an already-fetched repository tree. It prefers the most specific
// frontmatter-verified match (a SKILL.md whose directory matches the skill
// name), only considering the repository root when nothing deeper matches — so
// a repo-root SKILL.md whose name happens to collide with the skill name no
// longer captures the whole repository. When frontmatter matching finds nothing
// it falls back to accepting a conventional skill location by path (preserving
// the pre-tree importer's lenient semantics). When the tree is truncated it
// falls back to a bounded per-prefix listing.
func resolveSkillDirFromTree(ctx context.Context, httpClient *http.Client, owner, repo, defaultBranch, rawPrefix, skillName string, tree []githubTreeEntry, truncated bool) (string, []byte, error) {
skillPaths := extractSkillMdPaths(tree)
preferred, remaining := partitionSkillMdPaths(skillName, skillPaths)
if dir, body, ok := findMatchingSkillDirByFrontmatter(ctx, httpClient, rawPrefix, skillName, preferred); ok {
return dir, body, nil
}
if !truncated {
if dir, body, ok := findMatchingSkillDirByFrontmatter(ctx, httpClient, rawPrefix, skillName, remaining); ok {
return dir, body, nil
}
// Frontmatter matching found nothing. Fall back to path-based
// acceptance so a skill living at a conventional location
// (skills/<name>/SKILL.md, etc.) still imports even when its
// frontmatter name doesn't byte-match the URL slug — e.g. `name: Foo`
// for slug `foo`. This restores the pre-tree importer's semantics and
// matches what the rate-limited legacy fallback already accepts, using
// only the tree we already have (at most one extra raw fetch, never a
// directory crawl). Kept after the frontmatter pass so the bare repo
// root is never eligible here — the collision fix stays intact.
if dir, body, ok := acceptConventionalSkillDir(ctx, httpClient, rawPrefix, skillName, skillPaths, true); ok {
return dir, body, nil
}
return "", nil, skillMdNotFoundError(owner, repo, skillName)
}
slog.Warn("github import: repository tree listing truncated", "owner", owner, "repo", repo, "branch", defaultBranch)
if dir, body, ok := findSkillDirFromConventionalPrefixes(ctx, httpClient, owner, repo, defaultBranch, rawPrefix, skillName); ok {
return dir, body, nil
}
// Same path-based acceptance as the untruncated branch, so a conventional
// skill with a display-name frontmatter imports identically regardless of
// whether GitHub truncated the tree. The tree can't prove absence here, so
// the helper probes each candidate directly.
if dir, body, ok := acceptConventionalSkillDir(ctx, httpClient, rawPrefix, skillName, skillPaths, false); ok {
return dir, body, nil
}
return "", nil, fmt.Errorf("repository %s/%s tree is too large to scan exhaustively for skill %s", owner, repo, skillName)
}
// conventionalSkillMdPaths returns the SKILL.md locations the importer accepts
// by path (without a frontmatter name check) for a given skill slug. Order is
// significant: it mirrors the pre-tree importer's probe order.
func conventionalSkillMdPaths(skillName string) []string {
return []string{
"skills/" + skillName + "/SKILL.md",
".claude/skills/" + skillName + "/SKILL.md",
"plugin/skills/" + skillName + "/SKILL.md",
skillName + "/SKILL.md",
}
}
// acceptConventionalSkillDir accepts a skill by its conventional path
// (skills/<name>/SKILL.md, etc.) without requiring a frontmatter name match,
// restoring the pre-tree importer's lenient path-based acceptance. It never
// accepts the bare repo root, so a root SKILL.md whose name collides with the
// slug can only be selected through the frontmatter pass — the collision fix
// stays intact.
//
// When treeComplete is true, the untruncated tree proves which conventional
// paths exist, so absent candidates are skipped without a network call. When
// the tree was truncated it can't prove absence, so each candidate is probed
// directly (a missing candidate simply 404s and is skipped).
func acceptConventionalSkillDir(ctx context.Context, httpClient *http.Client, rawPrefix, skillName string, treeSkillPaths []string, treeComplete bool) (string, []byte, bool) {
present := make(map[string]struct{}, len(treeSkillPaths))
for _, p := range treeSkillPaths {
present[p] = struct{}{}
}
for _, candidate := range conventionalSkillMdPaths(skillName) {
if treeComplete {
if _, ok := present[candidate]; !ok {
continue // a complete tree proves this path is absent
}
}
body, err := fetchRawFile(ctx, httpClient, buildRawGitHubURL(rawPrefix, candidate))
if err != nil {
// With a complete tree the path was proven present, so a fetch
// error is unexpected and worth a breadcrumb; with a truncated tree
// this is just a probe miss.
if treeComplete {
slog.Warn("skills.sh import: conventional SKILL.md fetch failed", "path", candidate, "error", err)
}
continue
}
return skillDirFromSkillFilePath(candidate), body, true
}
return "", nil, false
}
// treeDownloadConcurrency bounds how many supporting files are downloaded in
// parallel during tree-based collection. Small enough that one import cannot
// open hundreds of sockets to GitHub at once, large enough to keep the download
// phase well under the overall import deadline for a full (256-file) bundle.
const treeDownloadConcurrency = 8
// addSupportingFilesFromTree enumerates the supporting files under skillDir from
// a single recursive git tree, enforces the per-file / count / total-byte caps
// arithmetically from the tree metadata BEFORE downloading anything (so an
// over-limit skill fails fast with a clear error instead of timing out), then
// downloads the surviving files concurrently and appends them in a stable path
// order. It replaces the legacy per-directory contents crawl (collectGitHubFiles).
func addSupportingFilesFromTree(ctx context.Context, httpClient *http.Client, result *importedSkill, tree []githubTreeEntry, rawPrefix, skillDir string) error {
basePath := ""
if skillDir != "" {
basePath = skillDir + "/"
}
// Select the eligible supporting-file blobs under skillDir, mirroring the
// filters the download loop / addFile would otherwise apply: skip the
// skill's own SKILL.md, LICENSE files, and binary assets (which addFile
// drops anyway). Keeping the filter here makes the arithmetic cap check
// below match what actually gets imported.
type treeFile struct {
repoPath string
relPath string
size int64
}
var eligible []treeFile
for _, entry := range tree {
if entry.Type != "blob" {
continue
}
if basePath != "" && !strings.HasPrefix(entry.Path, basePath) {
continue
}
relPath := strings.TrimPrefix(entry.Path, basePath)
if relPath == "" {
continue
}
lowerBase := strings.ToLower(filepath.Base(relPath))
if lowerBase == "skill.md" || lowerBase == "license" || lowerBase == "license.txt" || lowerBase == "license.md" {
continue
}
if isLikelyBinaryFilePath(relPath) {
continue
}
eligible = append(eligible, treeFile{repoPath: entry.Path, relPath: relPath, size: entry.size()})
}
// Stable order so imports are deterministic regardless of download timing.
sort.Slice(eligible, func(i, j int) bool { return eligible[i].relPath < eligible[j].relPath })
// Arithmetic cap pre-check on the tree metadata — no downloads required to
// reject an over-limit bundle.
if len(eligible) > maxImportFileCount {
return fmt.Errorf("%w: import bundle would contain %d files, exceeding the %d file limit", errImportCapExceeded, len(eligible), maxImportFileCount)
}
var totalSize int64
for _, f := range eligible {
if f.size > maxImportFileSize {
return fmt.Errorf("%w: %s is %d bytes, exceeding the %d byte per-file limit", errImportCapExceeded, f.relPath, f.size, maxImportFileSize)
}
totalSize += f.size
}
if totalSize > maxImportTotalSize {
return fmt.Errorf("%w: import bundle is %d bytes, exceeding the %d byte limit", errImportCapExceeded, totalSize, maxImportTotalSize)
}
// Download concurrently, then append in the pre-sorted order.
contents := make([]string, len(eligible))
fetched := make([]bool, len(eligible))
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(treeDownloadConcurrency)
for i, f := range eligible {
i, f := i, f
g.Go(func() error {
body, err := fetchRawFile(gctx, httpClient, buildRawGitHubURL(rawPrefix, f.repoPath))
if err != nil {
if isCapError(err) {
return fmt.Errorf("github import: %s: %w", f.repoPath, err)
}
// A cancelled context (overall import deadline exceeded, or the
// client disconnecting) must abort, not be swallowed as a
// per-file skip: otherwise every remaining download fails the
// same way, g.Wait returns nil, and the user gets a bundle that
// looks complete but is silently missing files. Let it surface
// so importFetchErrorResponse maps the deadline to a readable
// 504.
if gctx.Err() != nil {
return fmt.Errorf("github import: fetch aborted at %s: %w", f.repoPath, gctx.Err())
}
// Otherwise match the legacy loop's leniency: a single failed
// supporting file is logged and skipped, not fatal.
slog.Warn("github import: file download failed", "path", f.repoPath, "error", err)
return nil
}
contents[i] = string(body)
fetched[i] = true
return nil
})
}
if err := g.Wait(); err != nil {
return err
}
for i, f := range eligible {
if !fetched[i] {
continue
}
if err := result.addFile(f.relPath, contents[i]); err != nil {
return err
}
}
return nil
}
// size returns the blob byte size, guarding against a negative value from a
// malformed tree response.
func (e githubTreeEntry) size() int64 {
if e.Size < 0 {
return 0
}
return e.Size
}
// collectGitHubFiles recursively collects file entries from a GitHub directory listing.
func collectGitHubFiles(ctx context.Context, httpClient *http.Client, entries []githubContentEntry, out *[]githubContentEntry, parentURL string) {
for _, entry := range entries {
// Stop descending once the context is cancelled; the caller checks
// ctx.Err() afterwards and aborts rather than importing a partial crawl.
if ctx.Err() != nil {
return
}
lower := strings.ToLower(entry.Name)
if lower == "skill.md" || lower == "license" || lower == "license.txt" || lower == "license.md" {
continue
}
if entry.Type == "file" {
*out = append(*out, entry)
} else if entry.Type == "dir" {
// Fetch subdirectory contents
subURL := entry.URL
if subURL == "" {
parsed, err := url.Parse(parentURL)
if err != nil {
slog.Warn("github import: invalid parent directory url", "url", parentURL, "error", err)
continue
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/" + entry.Name
subURL = parsed.String()
}
subResp, err := doGitHubAPIGet(ctx, httpClient, subURL)
if err != nil || subResp.StatusCode != http.StatusOK {
attrs := []any{"url", subURL}
if subResp != nil {
attrs = append(attrs, "status", subResp.StatusCode)
subResp.Body.Close()
}
if err != nil {
attrs = append(attrs, "error", err)
}
slog.Warn("github import: failed to list subdirectory", attrs...)
continue
}
var subEntries []githubContentEntry
if err := json.NewDecoder(subResp.Body).Decode(&subEntries); err != nil {
subResp.Body.Close()
slog.Warn("github import: failed to decode subdirectory listing", "url", subURL, "error", err)
continue
}
subResp.Body.Close()
collectGitHubFiles(ctx, httpClient, subEntries, out, subURL)
}
}
}
func findSkillDirFromConventionalPrefixes(ctx context.Context, httpClient *http.Client, owner, repo, defaultBranch, rawPrefix, skillName string) (string, []byte, bool) {
prefixes := []string{"skills", ".claude/skills", "plugin/skills"}
var skillPaths []string
for _, prefix := range prefixes {
paths, err := listGitHubSkillMdPaths(ctx, httpClient, owner, repo, prefix, defaultBranch)
if err != nil {
slog.Warn("github import: failed to list conventional skill prefix", "prefix", prefix, "error", err)
continue
}
skillPaths = append(skillPaths, paths...)
}
preferred, remaining := partitionSkillMdPaths(skillName, skillPaths)
if dir, body, ok := findMatchingSkillDirByFrontmatter(ctx, httpClient, rawPrefix, skillName, preferred); ok {
return dir, body, true
}
return findMatchingSkillDirByFrontmatter(ctx, httpClient, rawPrefix, skillName, remaining)
}
func listGitHubSkillMdPaths(ctx context.Context, httpClient *http.Client, owner, repo, repoPath, ref string) ([]string, error) {
apiURL := buildGitHubContentsURL(owner, repo, repoPath, ref)
resp, err := doGitHubAPIGet(ctx, httpClient, apiURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
var entries []githubContentEntry
if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil {
return nil, err
}
var paths []string
collectGitHubSkillMdPaths(ctx, httpClient, entries, &paths, apiURL)
return paths, nil
}
func collectGitHubSkillMdPaths(ctx context.Context, httpClient *http.Client, entries []githubContentEntry, out *[]string, parentURL string) {
for _, entry := range entries {
lower := strings.ToLower(entry.Name)
if entry.Type == "file" {
if lower == "skill.md" {
*out = append(*out, entry.Path)
}
continue
}
if entry.Type != "dir" {
continue
}
subURL := entry.URL
if subURL == "" {
parsed, err := url.Parse(parentURL)
if err != nil {
slog.Warn("github import: invalid parent directory url", "url", parentURL, "error", err)
continue
}
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/" + entry.Name
subURL = parsed.String()
}
subResp, err := doGitHubAPIGet(ctx, httpClient, subURL)
if err != nil || subResp.StatusCode != http.StatusOK {
attrs := []any{"url", subURL}
if subResp != nil {
attrs = append(attrs, "status", subResp.StatusCode)
subResp.Body.Close()
}
if err != nil {
attrs = append(attrs, "error", err)
}
slog.Warn("github import: failed to list skill metadata subdirectory", attrs...)
continue
}
var subEntries []githubContentEntry
if err := json.NewDecoder(subResp.Body).Decode(&subEntries); err != nil {
subResp.Body.Close()
slog.Warn("github import: failed to decode skill metadata subdirectory", "url", subURL, "error", err)
continue
}
subResp.Body.Close()
collectGitHubSkillMdPaths(ctx, httpClient, subEntries, out, subURL)
}
}
func extractSkillMdPaths(entries []githubTreeEntry) []string {
paths := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.Type != "blob" || (!strings.HasSuffix(entry.Path, "/SKILL.md") && entry.Path != "SKILL.md") {
continue
}
paths = append(paths, entry.Path)
}
return paths
}
func partitionSkillMdPaths(skillName string, skillPaths []string) (preferred []string, remaining []string) {
for _, skillPath := range skillPaths {
if isLikelySkillPathMatch(skillName, skillPath) {
preferred = append(preferred, skillPath)
continue
}
remaining = append(remaining, skillPath)
}
return preferred, remaining
}
func findMatchingSkillDirByFrontmatter(ctx context.Context, httpClient *http.Client, rawPrefix, skillName string, skillPaths []string) (string, []byte, bool) {
for _, skillPath := range skillPaths {
body, err := fetchRawFile(ctx, httpClient, buildRawGitHubURL(rawPrefix, skillPath))
if err != nil {
slog.Warn("github import: fallback SKILL.md fetch failed", "path", skillPath, "error", err)
continue
}
name, _ := skillpkg.ParseSkillFrontmatter(string(body))
if name == skillName {
return skillDirFromSkillFilePath(skillPath), body, true
}
}
return "", nil, false
}
func isLikelySkillPathMatch(skillName, skillPath string) bool {
dir := strings.ToLower(skillDirFromSkillFilePath(skillPath))
base := strings.ToLower(filepath.Base(dir))
for _, hint := range skillNameHints(skillName) {
if strings.Contains(dir, hint) || strings.Contains(base, hint) || strings.Contains(hint, base) {
return true
}
}
return false
}
func skillNameHints(skillName string) []string {
skillName = strings.ToLower(skillName)
parts := strings.Split(skillName, "-")
seen := map[string]struct{}{}
var hints []string
addHint := func(value string) {
value = strings.TrimSpace(value)
if len(value) < 3 {
return
}
if _, ok := seen[value]; ok {
return
}
seen[value] = struct{}{}
hints = append(hints, value)
}
addHint(skillName)
for i := 1; i < len(parts); i++ {
addHint(strings.Join(parts[i:], "-"))
}
for _, part := range parts {
addHint(part)
}
return hints
}
// --- GitHub import ---
// errGitHubAPIBlocked signals that an api.github.com probe was rejected for
// auth/rate-limit reasons (401/403/429) rather than because the resource
// genuinely does not exist. Resolvers treat this as "indeterminate" and may
// fall back to the optimistic URL split rather than aborting the import.
var errGitHubAPIBlocked = errors.New("github API blocked (rate limit or auth)")
// doGitHubAPIGet performs a GET against an api.github.com URL, attaching the
// GITHUB_TOKEN bearer header when the env var is set. Unauthenticated GitHub
// API requests are capped at 60/hour per IP, which is trivially exhausted on
// shared self-hosted servers and surfaces to users as 403 errors during
// skill imports. Setting GITHUB_TOKEN raises the limit to 5000/hour.
func doGitHubAPIGet(ctx context.Context, httpClient *http.Client, apiURL string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
if err != nil {
return nil, err
}
addGitHubAuthHeader(req)
return httpClient.Do(req)
}
func addGitHubAuthHeader(req *http.Request) {
if req == nil {
return
}
if token := strings.TrimSpace(os.Getenv("GITHUB_TOKEN")); token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
}
// githubSpec captures the parsed components of a github.com URL pointing at a
// skill (or single-skill repository).
type githubSpec struct {
owner string
repo string
ref string // empty → caller resolves the default branch
skillDir string // relative directory within the repo, "" for the repository root
// refSegments holds the raw path segments after /tree/ or /blob/ that
// jointly encode (ref, skillDir). GitHub's web URLs do not delimit the
// boundary between branch/tag name and in-repo path, so when a ref
// contains '/' (e.g. "release/v2") segments[0] alone is not the ref.
// fetchFromGitHub uses resolveGitHubRefAndPath to walk these segments
// and ask the API which prefix is a real branch/tag/commit. When this
// slice is empty, ref/skillDir above are authoritative (root URL).
refSegments []string
// kind is "tree" or "blob"; "" for root URLs. blob requires the last
// segment to be SKILL.md, which is already stripped from refSegments.
kind string
}
// parseGitHubURL extracts the owner, repo, and the raw post-/tree|/blob
// segments from a github.com URL. Supported forms:
//
// github.com/{owner}/{repo} → root, default branch
// github.com/{owner}/{repo}/tree/{ref}/{path...} → ref / skill dir
// github.com/{owner}/{repo}/blob/{ref}/{path.../SKILL.md} → ref / skill dir
//
// A simple-ref shortcut (segments[0] is the ref, the rest is the path) is
// stored in spec.ref/spec.skillDir; refSegments is also populated so that
// fetchFromGitHub can disambiguate refs containing '/' against the API.
func parseGitHubURL(raw string) (githubSpec, error) {
parsed, err := url.Parse(raw)
if err != nil {
return githubSpec{}, fmt.Errorf("invalid URL: %w", err)
}
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
return githubSpec{}, fmt.Errorf("expected URL format: github.com/{owner}/{repo}[/tree/{ref}/{path}], got: %s", parsed.Path)
}
spec := githubSpec{owner: parts[0], repo: strings.TrimSuffix(parts[1], ".git")}
if len(parts) == 2 {
return spec, nil
}
kind := parts[2]
if kind != "tree" && kind != "blob" {
return githubSpec{}, fmt.Errorf("unsupported URL form: github.com/%s/%s/%s/... (use /tree/{ref}/... or /blob/{ref}/.../SKILL.md)", spec.owner, spec.repo, kind)
}
if len(parts) < 4 || parts[3] == "" {
return githubSpec{}, fmt.Errorf("missing ref after /%s/", kind)
}
spec.kind = kind
rest := parts[3:]
if kind == "blob" {
if !strings.EqualFold(rest[len(rest)-1], "SKILL.md") {
return githubSpec{}, fmt.Errorf("blob URL must point to a SKILL.md file")
}
rest = rest[:len(rest)-1]
if len(rest) == 0 {
return githubSpec{}, fmt.Errorf("missing ref after /blob/")
}
}
// Decode URL-escaped segments (e.g. spaces) so paths match the repo's
// real on-disk layout. Re-escaping happens in buildRawGitHubURL.
decoded := make([]string, len(rest))
for i, p := range rest {
d, err := url.PathUnescape(p)
if err != nil {
return githubSpec{}, fmt.Errorf("invalid path segment %q: %w", p, err)
}
if d == "" {
return githubSpec{}, fmt.Errorf("empty path segment in URL")
}
decoded[i] = d
}
spec.refSegments = decoded
// Optimistic split: assume the simple case where the ref is one segment.
// fetchFromGitHub will re-resolve via the API and overwrite both fields
// when the optimistic guess does not validate (e.g. release/v2 refs).
spec.ref = decoded[0]
if len(decoded) > 1 {
spec.skillDir = strings.Join(decoded[1:], "/")
}
return spec, nil
}
// resolveGitHubRefAndPath walks the parsed refSegments and asks the GitHub
// commits API which prefix corresponds to a real branch, tag, or commit.
// This is what makes refs containing '/' (e.g. "release/v2") work correctly:
// the URL github.com/o/r/tree/release/v2/skills/foo is ambiguous between
// (ref=release, path=v2/skills/foo) and (ref=release/v2, path=skills/foo),
// so we probe /repos/{o}/{r}/commits/{candidate} from longest to shortest
// and accept the first one the server confirms exists.
//
// On success spec.ref and spec.skillDir are overwritten with the resolved
// pair. On failure (no candidate resolves) a single error is returned that
// names every candidate that was tried.
func resolveGitHubRefAndPath(ctx context.Context, httpClient *http.Client, spec *githubSpec) error {
if len(spec.refSegments) == 0 {
return nil
}
// Try longest prefix first so that release/v2 wins over release.
tried := make([]string, 0, len(spec.refSegments))
blocked := false
for n := len(spec.refSegments); n >= 1; n-- {
candidate := strings.Join(spec.refSegments[:n], "/")
tried = append(tried, candidate)
ok, err := githubRefExists(ctx, httpClient, spec.owner, spec.repo, candidate)
if errors.Is(err, errGitHubAPIBlocked) {
// 401/403/429 means we can't tell whether the ref exists. Keep
// trying the remaining (shorter) candidates so we don't punish
// the common single-segment-ref case for one bad probe.
blocked = true
continue
}
if err != nil {
// Network / transport errors should not be silently treated as
// "ref does not exist" — surface them so the caller can retry.
return fmt.Errorf("validating ref %q: %w", candidate, err)
}
if ok {
spec.ref = candidate
if n == len(spec.refSegments) {
spec.skillDir = ""
} else {
spec.skillDir = strings.Join(spec.refSegments[n:], "/")
}
return nil
}
}
if blocked {
// Every probe was either a confirmed 404 or rate-limited and we never
// got a confirmation. Fall back to the optimistic single-segment
// split that parseGitHubURL populated. If that's wrong, the
// subsequent raw-file fetch will surface a clearer "SKILL.md not
// found" error than failing the whole import on a 403.
slog.Warn("github import: ref resolution blocked by GitHub API (rate limit or auth); falling back to optimistic single-segment ref. Set GITHUB_TOKEN to enable disambiguation of slash-bearing refs.",
"owner", spec.owner, "repo", spec.repo, "tried", tried)
return nil
}
return fmt.Errorf("could not resolve ref in github.com/%s/%s URL — tried: %s. Make sure the branch, tag, or commit exists and that the URL is the canonical /tree/{ref}/{path} or /blob/{ref}/{path}/SKILL.md form",
spec.owner, spec.repo, strings.Join(tried, ", "))
}
// githubRefExists returns true when GitHub recognizes ref as a branch, tag,
// or commit SHA on owner/repo. It uses the commits endpoint because that
// single call accepts all three ref kinds (unlike /branches or /tags which
// only match one). 404 means the ref does not exist; any other non-200
// status is treated as an error so the caller can distinguish "missing"
// from "API down".
func githubRefExists(ctx context.Context, httpClient *http.Client, owner, repo, ref string) (bool, error) {
apiURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/commits/%s",
url.PathEscape(owner), url.PathEscape(repo), escapeRefPath(ref))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
if err != nil {
return false, err
}
// Per GitHub docs: Accept: application/vnd.github.v3.sha returns just
// the SHA when the ref resolves, which is the cheapest possible probe.
req.Header.Set("Accept", "application/vnd.github.v3.sha")
addGitHubAuthHeader(req)
resp, err := httpClient.Do(req)
if err != nil {
return false, err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound, http.StatusUnprocessableEntity:
return false, nil
case http.StatusUnauthorized, http.StatusForbidden, http.StatusTooManyRequests:
return false, errGitHubAPIBlocked
default:
return false, fmt.Errorf("github API returned status %d for ref %q", resp.StatusCode, ref)
}
}
func fetchFromGitHub(ctx context.Context, httpClient *http.Client, rawURL string) (*importedSkill, error) {
spec, err := parseGitHubURL(rawURL)
if err != nil {
return nil, err
}
if len(spec.refSegments) > 0 {
// Disambiguate slash-bearing refs (release/v2 etc.) against the API
// before issuing any raw or contents requests.
if err := resolveGitHubRefAndPath(ctx, httpClient, &spec); err != nil {
return nil, err
}
}
if spec.ref == "" {
spec.ref = fetchGitHubDefaultBranch(ctx, httpClient, spec.owner, spec.repo)
}
rawPrefix := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s",
url.PathEscape(spec.owner), url.PathEscape(spec.repo), escapeRefPath(spec.ref))
skillMdPath := "SKILL.md"
if spec.skillDir != "" {
skillMdPath = spec.skillDir + "/SKILL.md"
}
skillMdBody, err := fetchRawFile(ctx, httpClient, buildRawGitHubURL(rawPrefix, skillMdPath))
if err != nil {
if spec.skillDir == "" {
return nil, fmt.Errorf("SKILL.md not found at the root of %s/%s@%s. For multi-skill repositories, point to a specific directory using github.com/%s/%s/tree/%s/<skill-dir>",
spec.owner, spec.repo, spec.ref, spec.owner, spec.repo, spec.ref)
}
return nil, fmt.Errorf("SKILL.md not found at %s in %s/%s@%s: %w",
skillMdPath, spec.owner, spec.repo, spec.ref, err)
}
name, description := skillpkg.ParseSkillFrontmatter(string(skillMdBody))
if name == "" {
if spec.skillDir != "" {
name = filepath.Base(spec.skillDir)
} else {
name = spec.repo
}
}
result := &importedSkill{
name: name,
description: description,
content: string(skillMdBody),
origin: map[string]any{
"type": "github",
"source_url": rawURL,
"owner": spec.owner,
"repo": spec.repo,
"ref": spec.ref,
"path": spec.skillDir,
},
}
// Enumerate supporting files from a single recursive tree, checking the
// import caps against the tree metadata before downloading anything. Fall
// back to the per-directory contents crawl when the tree is unavailable or
// truncated (kept lenient so a rate-limited listing doesn't fail an import
// that already produced a valid SKILL.md).
tree, truncated, treeErr := fetchGitHubTree(ctx, httpClient, spec.owner, spec.repo, spec.ref)
if treeErr == nil && !truncated {
if err := addSupportingFilesFromTree(ctx, httpClient, result, tree, rawPrefix, spec.skillDir); err != nil {
return nil, err
}
return result, nil
}
if err := addSupportingFilesViaCrawl(ctx, httpClient, result, spec.owner, spec.repo, spec.ref, spec.skillDir); err != nil {
return nil, err
}
return result, nil
}
// --- Shared helpers ---
// rawGitHubContentHost serves raw GitHub file content. fetchRawFile attaches the
// GITHUB_TOKEN only for this host: the same function downloads files from
// non-GitHub skill sources (clawhub.ai, skills.sh), and an unconditional auth
// header would leak the token to those third-party hosts.
const rawGitHubContentHost = "raw.githubusercontent.com"
// fetchRawFile downloads a URL and returns the body bytes. Returns an error
// if the response exceeds maxImportFileSize so we never silently truncate a
// half-downloaded skill file into the workspace.
func fetchRawFile(ctx context.Context, httpClient *http.Client, fileURL string) ([]byte, error) {
req, err := newRawFileRequest(ctx, fileURL)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxImportFileSize+1))
if err != nil {
return nil, err
}
if len(body) > maxImportFileSize {
return nil, fmt.Errorf("%w: file exceeds %d byte limit", errImportCapExceeded, maxImportFileSize)
}
return body, nil
}
// newRawFileRequest builds the GET request for a raw skill file, attaching the
// GitHub auth header only when the URL targets GitHub's raw content host. The
// host gate lives here, separate from the round-trip, so it can be unit tested
// without a live network call — GITHUB_TOKEN must never reach a non-GitHub
// skill host.
func newRawFileRequest(ctx context.Context, fileURL string) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
if err != nil {
return nil, err
}
if strings.EqualFold(req.URL.Hostname(), rawGitHubContentHost) {
addGitHubAuthHeader(req)
}
return req, nil
}
// escapeRefPath percent-encodes each segment of a git ref individually so
// that slash-bearing refs like "release/v2" are sent to GitHub as
// "release/v2" (path separators preserved) rather than "release%2Fv2"
// (which GitHub does not accept on the commits / raw endpoints).
func escapeRefPath(ref string) string {
parts := strings.Split(ref, "/")
for i, p := range parts {
parts[i] = url.PathEscape(p)
}
return strings.Join(parts, "/")
}
func buildRawGitHubURL(rawPrefix, repoPath string) string {
parts := strings.Split(strings.Trim(repoPath, "/"), "/")
escaped := make([]string, 0, len(parts))
for _, part := range parts {
if part == "" {
continue
}
escaped = append(escaped, url.PathEscape(part))
}
if len(escaped) == 0 {
return rawPrefix
}
return rawPrefix + "/" + strings.Join(escaped, "/")
}
func buildGitHubContentsURL(owner, repo, repoPath, ref string) string {
base := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents",
url.PathEscape(owner), url.PathEscape(repo))
if repoPath == "" {
return base + "?ref=" + url.QueryEscape(ref)
}
return base + "/" + strings.TrimPrefix(buildRawGitHubURL("", repoPath), "/") + "?ref=" + url.QueryEscape(ref)
}
func skillDirFromSkillFilePath(path string) string {
if path == "SKILL.md" {
return ""
}
return strings.TrimSuffix(path, "/SKILL.md")
}
func skillMdNotFoundError(owner, repo, skillName string) error {
return fmt.Errorf("SKILL.md not found in repository %s/%s for skill %s", owner, repo, skillName)
}
func skillImportConflictReason() string {
return "a skill with this name already exists; use --on-conflict overwrite to replace it or --on-conflict rename to import a copy"
}
func (h *Handler) createImportedSkillWithName(ctx context.Context, workspaceID, creatorID pgtype.UUID, name string, imported *importedSkill, config map[string]any, files []CreateSkillFileRequest) (SkillWithFilesResponse, error) {
return h.createSkillWithFiles(ctx, skillCreateInput{
WorkspaceID: workspaceID,
CreatorID: creatorID,
Name: name,
Description: imported.description,
Content: imported.content,
Config: config,
Files: files,
})
}
func (h *Handler) createRenamedImportedSkill(ctx context.Context, workspaceID, creatorID pgtype.UUID, baseName string, imported *importedSkill, config map[string]any, files []CreateSkillFileRequest) (SkillWithFilesResponse, error) {
for suffix := 2; suffix < maxImportRenameAttempts+2; suffix++ {
candidate := fmt.Sprintf("%s-%d", baseName, suffix)
resp, err := h.createImportedSkillWithName(ctx, workspaceID, creatorID, candidate, imported, config, files)
if err == nil {
return resp, nil
}
if !isUniqueViolation(err) {
return SkillWithFilesResponse{}, err
}
}
return SkillWithFilesResponse{}, fmt.Errorf("failed to find an available renamed skill name after %d attempts", maxImportRenameAttempts)
}
func skillImportOverwriteFailure(err error) (int, string) {
switch {
case errors.Is(err, errSkillOverwriteNotFound):
return http.StatusConflict, "target skill no longer exists"
case errors.Is(err, errSkillOverwriteForbidden):
return http.StatusForbidden, "only the skill creator can overwrite this skill"
case errors.Is(err, errSkillOverwriteNameMismatch):
return http.StatusConflict, "target skill name no longer matches the imported skill"
default:
return http.StatusInternalServerError, "failed to overwrite skill: " + err.Error()
}
}
func (h *Handler) resolveImportSkillConflict(w http.ResponseWriter, r *http.Request, strategy string, workspaceID string, workspaceUUID, creatorUUID pgtype.UUID, creatorID string, name string, imported *importedSkill, config map[string]any, files []CreateSkillFileRequest, existing db.Skill) {
existingInfo := existingSkillIdentity(existing, creatorID)
switch strategy {
case importOnConflictSkip:
writeJSON(w, http.StatusOK, SkillImportResult{
Status: "skipped",
Reason: "a skill with this name already exists",
ExistingSkill: &existingInfo,
})
case importOnConflictOverwrite:
if !canOverwriteSkillByLocalImport(creatorID, existing) {
writeJSON(w, http.StatusForbidden, SkillImportResult{
Status: "failed",
Reason: "only the skill creator can overwrite this skill",
ExistingSkill: &existingInfo,
})
return
}
resp, err := h.overwriteSkillWithFiles(r.Context(), skillOverwriteInput{
WorkspaceID: workspaceUUID,
TargetSkillID: existing.ID,
UserID: creatorID,
ExpectedName: name,
Description: imported.description,
Content: imported.content,
Config: config,
Files: files,
})
if err != nil {
status, reason := skillImportOverwriteFailure(err)
writeJSON(w, status, SkillImportResult{
Status: "failed",
Reason: reason,
ExistingSkill: &existingInfo,
})
return
}
actorType, actorID := h.resolveActor(r, creatorID, workspaceID)
h.publish(protocol.EventSkillUpdated, workspaceID, actorType, actorID, map[string]any{"skill": resp})
writeJSON(w, http.StatusOK, SkillImportResult{Status: "updated", Skill: &resp})
case importOnConflictRename:
resp, err := h.createRenamedImportedSkill(r.Context(), workspaceUUID, creatorUUID, name, imported, config, files)
if err != nil {
writeJSON(w, http.StatusInternalServerError, SkillImportResult{
Status: "failed",
Reason: "failed to create renamed skill: " + err.Error(),
ExistingSkill: &existingInfo,
})
return
}
actorType, actorID := h.resolveActor(r, creatorID, workspaceID)
h.publish(protocol.EventSkillCreated, workspaceID, actorType, actorID, map[string]any{"skill": resp})
writeJSON(w, http.StatusCreated, SkillImportResult{
Status: "created",
Reason: "renamed to avoid an existing skill",
Skill: &resp,
ExistingSkill: &existingInfo,
})
default:
writeJSON(w, http.StatusConflict, SkillImportResult{
Status: "conflict",
Reason: skillImportConflictReason(),
ExistingSkill: &existingInfo,
})
}
}
// --- Import handler ---
func (h *Handler) ImportSkill(w http.ResponseWriter, r *http.Request) {
workspaceID := h.resolveWorkspaceID(r)
creatorID, ok := requireUserID(w, r)
if !ok {
return
}
workspaceUUID, ok := parseUUIDOrBadRequest(w, workspaceID, "workspace_id")
if !ok {
return
}
creatorUUID := parseUUID(creatorID)
// An uploaded skill archive (.skill / .zip) arrives as multipart/form-data;
// a hosted-URL import arrives as JSON. Both converge on the same create +
// conflict tail via finishSkillImport.
if isMultipartForm(r) {
h.importSkillFromArchive(w, r, workspaceID, workspaceUUID, creatorUUID, creatorID)
return
}
var req ImportSkillRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if !validImportOnConflict(req.OnConflict) {
writeError(w, http.StatusBadRequest, "on_conflict must be one of: fail, overwrite, rename, skip")
return
}
structuredResult := req.OnConflict != ""
strategy := req.OnConflict
if strategy == "" {
strategy = importOnConflictFail
}
source, normalized, err := detectImportSource(req.URL)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
httpClient := &http.Client{Timeout: 30 * time.Second}
// Bound the whole server-side fetch under an overall deadline that is
// shorter than the reverse-proxy / CDN gateway timeout in front of the API.
// If an upstream is slow, the import returns a clear error instead of the
// proxy severing the connection with a 504, and a client disconnect cancels
// the in-flight fetch instead of letting it run on orphaned.
ctx, cancel := context.WithTimeout(r.Context(), importFetchTimeout)
defer cancel()
var imported *importedSkill
switch source {
case sourceClawHub:
imported, err = fetchFromClawHub(ctx, httpClient, normalized)
case sourceSkillsSh:
imported, err = fetchFromSkillsSh(ctx, httpClient, normalized)
case sourceGitHub:
imported, err = fetchFromGitHub(ctx, httpClient, normalized)
}
if err != nil {
status, msg := importFetchErrorResponse(ctx, err)
writeError(w, status, msg)
return
}
h.finishSkillImport(w, r, workspaceID, workspaceUUID, creatorUUID, creatorID, strategy, structuredResult, imported)
}
// importFetchTimeout bounds the total time spent fetching a skill's files from
// the upstream source. It is deliberately below the reverse-proxy gateway
// timeout so a slow or oversized source surfaces as a readable API error rather
// than a proxy 504.
const importFetchTimeout = 45 * time.Second
// importFetchErrorResponse maps a fetch failure onto an HTTP status and message.
// Cap violations become 413 (the skill is too large to import), an exhausted
// deadline becomes 504, a transiently unavailable source becomes a retryable
// 503, and everything else stays 502.
func importFetchErrorResponse(ctx context.Context, err error) (int, string) {
if isCapError(err) {
return http.StatusRequestEntityTooLarge, err.Error()
}
if errors.Is(err, context.DeadlineExceeded) || ctx.Err() == context.DeadlineExceeded {
return http.StatusGatewayTimeout, "skill import timed out fetching source files; the skill may be too large or the source too slow"
}
if errors.Is(err, errImportSourceUnavailable) {
return http.StatusServiceUnavailable, err.Error()
}
return http.StatusBadGateway, err.Error()
}
// finishSkillImport runs the shared tail of every skill import — whether the
// bundle came from a hosted URL or an uploaded archive (.skill / .zip). It maps
// the extracted files onto CreateSkillFileRequest, records provenance into
// config.origin, and creates the skill, routing same-name collisions through
// the on_conflict strategy.
func (h *Handler) finishSkillImport(w http.ResponseWriter, r *http.Request, workspaceID string, workspaceUUID, creatorUUID pgtype.UUID, creatorID, strategy string, structuredResult bool, imported *importedSkill) {
files := make([]CreateSkillFileRequest, 0, len(imported.files))
for _, f := range imported.files {
if !validateFilePath(f.path) {
continue
}
files = append(files, CreateSkillFileRequest{
Path: f.path,
Content: f.content,
})
}
// Persist provenance into skill.config.origin so list/detail UI can show
// "Imported from GitHub / ClawHub / Skills.sh" and link back to the source.
config := map[string]any{}
if imported.origin != nil {
config["origin"] = imported.origin
}
name := sanitizeNullBytes(imported.name)
if structuredResult {
if existing, found, lerr := h.lookupSkillByName(r.Context(), workspaceUUID, name); lerr != nil {
writeJSON(w, http.StatusInternalServerError, SkillImportResult{
Status: "failed",
Reason: "failed to check for existing skill: " + lerr.Error(),
})
return
} else if found {
h.resolveImportSkillConflict(w, r, strategy, workspaceID, workspaceUUID, creatorUUID, creatorID, name, imported, config, files, existing)
return
}
}
resp, err := h.createImportedSkillWithName(r.Context(), workspaceUUID, creatorUUID, name, imported, config, files)
if err != nil {
if isUniqueViolation(err) {
if structuredResult {
if existing, found, lerr := h.lookupSkillByName(r.Context(), workspaceUUID, name); lerr == nil && found {
h.resolveImportSkillConflict(w, r, strategy, workspaceID, workspaceUUID, creatorUUID, creatorID, name, imported, config, files, existing)
return
}
}
if existing, found, findErr := h.existingSkillIdentityByName(r.Context(), workspaceUUID, name); findErr == nil && found {
writeSkillImportDuplicateConflict(w, existing)
} else {
writeError(w, http.StatusConflict, "a skill with this name already exists")
}
return
}
writeError(w, http.StatusInternalServerError, "failed to create skill: "+err.Error())
return
}
actorType, actorID := h.resolveActor(r, creatorID, workspaceID)
h.publish(protocol.EventSkillCreated, workspaceID, actorType, actorID, map[string]any{"skill": resp})
if structuredResult {
writeJSON(w, http.StatusCreated, SkillImportResult{Status: "created", Skill: &resp})
return
}
writeJSON(w, http.StatusCreated, resp)
}
// --- Skill File endpoints ---
func (h *Handler) ListSkillFiles(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
skill, ok := h.loadSkillForUser(w, r, id)
if !ok {
return
}
files, err := h.Queries.ListSkillFiles(r.Context(), skill.ID)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to list skill files")
return
}
resp := make([]SkillFileResponse, len(files))
for i, f := range files {
resp[i] = skillFileToResponse(f)
}
writeJSON(w, http.StatusOK, resp)
}
func (h *Handler) UpsertSkillFile(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
skill, ok := h.loadSkillForUser(w, r, id)
if !ok {
return
}
if !h.canManageSkill(w, r, skill) {
return
}
var req CreateSkillFileRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if !validateFilePath(req.Path) {
writeError(w, http.StatusBadRequest, "invalid file path")
return
}
if skillpkg.IsReservedContentPath(req.Path) {
writeError(w, http.StatusBadRequest, "SKILL.md is reserved for the primary skill content")
return
}
sf, err := h.Queries.UpsertSkillFile(r.Context(), db.UpsertSkillFileParams{
SkillID: skill.ID,
Path: sanitizeNullBytes(req.Path),
Content: sanitizeNullBytes(req.Content),
})
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to upsert skill file: "+err.Error())
return
}
writeJSON(w, http.StatusOK, skillFileToResponse(sf))
}
func (h *Handler) DeleteSkillFile(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
skill, ok := h.loadSkillForUser(w, r, id)
if !ok {
return
}
if !h.canManageSkill(w, r, skill) {
return
}
fileID := chi.URLParam(r, "fileId")
fileUUID, ok := parseUUIDOrBadRequest(w, fileID, "file id")
if !ok {
return
}
// Verify the file belongs to the parent skill we just authorized — guards
// against deleting a file owned by a different skill via the URL param.
file, err := h.Queries.GetSkillFile(r.Context(), fileUUID)
if err != nil || uuidToString(file.SkillID) != uuidToString(skill.ID) {
writeError(w, http.StatusNotFound, "skill file not found")
return
}
if err := h.Queries.DeleteSkillFile(r.Context(), file.ID); err != nil {
writeError(w, http.StatusInternalServerError, "failed to delete skill file")
return
}
w.WriteHeader(http.StatusNoContent)
}
// --- Agent-Skill junction ---
func (h *Handler) ListAgentSkills(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
agent, ok := h.loadAgentForUser(w, r, id)
if !ok {
return
}
skills, err := h.Queries.ListAgentSkillSummaries(r.Context(), agent.ID)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to list agent skills")
return
}
resp := make([]SkillSummaryResponse, len(skills))
for i, s := range skills {
resp[i] = skillSummaryToResponse(
s.ID, s.WorkspaceID, s.Name, s.Description, s.Config,
s.CreatedBy, s.CreatedAt, s.UpdatedAt,
)
resp[i].Enabled = &s.Enabled
}
writeJSON(w, http.StatusOK, resp)
}
func (h *Handler) SetAgentSkills(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
agent, ok := h.loadAgentForUser(w, r, id)
if !ok {
return
}
if !h.canManageAgent(w, r, agent) {
return
}
var req SetAgentSkillsRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
skillUUIDs, ok := parseUUIDSliceOrBadRequest(w, req.SkillIDs, "skill_ids")
if !ok {
return
}
if !h.validateAgentSkillIDsInWorkspace(w, r, agent, skillUUIDs) {
return
}
tx, err := h.TxStarter.Begin(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to start transaction")
return
}
defer tx.Rollback(r.Context())
qtx := h.Queries.WithTx(tx)
if err := qtx.RemoveAllAgentSkills(r.Context(), agent.ID); err != nil {
writeError(w, http.StatusInternalServerError, "failed to clear agent skills")
return
}
for _, skillID := range skillUUIDs {
if err := qtx.AddAgentSkill(r.Context(), db.AddAgentSkillParams{
AgentID: agent.ID,
SkillID: skillID,
}); err != nil {
writeError(w, http.StatusInternalServerError, "failed to add agent skill: "+err.Error())
return
}
}
if err := tx.Commit(r.Context()); err != nil {
writeError(w, http.StatusInternalServerError, "failed to commit")
return
}
h.writeUpdatedAgentSkills(w, r, agent)
}
func (h *Handler) AddAgentSkills(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
agent, ok := h.loadAgentForUser(w, r, id)
if !ok {
return
}
if !h.canManageAgent(w, r, agent) {
return
}
var req AddAgentSkillsRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
skillUUIDs, ok := parseUUIDSliceOrBadRequest(w, req.SkillIDs, "skill_ids")
if !ok {
return
}
if !h.validateAgentSkillIDsInWorkspace(w, r, agent, skillUUIDs) {
return
}
tx, err := h.TxStarter.Begin(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to start transaction")
return
}
defer tx.Rollback(r.Context())
qtx := h.Queries.WithTx(tx)
for _, skillID := range skillUUIDs {
if err := qtx.AddAgentSkill(r.Context(), db.AddAgentSkillParams{
AgentID: agent.ID,
SkillID: skillID,
}); err != nil {
writeError(w, http.StatusInternalServerError, "failed to add agent skill: "+err.Error())
return
}
}
if err := tx.Commit(r.Context()); err != nil {
writeError(w, http.StatusInternalServerError, "failed to commit")
return
}
h.writeUpdatedAgentSkills(w, r, agent)
}
func (h *Handler) SetAgentSkillEnabled(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
agent, ok := h.loadAgentForUser(w, r, id)
if !ok {
return
}
if !h.canManageAgent(w, r, agent) {
return
}
skillID, ok := parseUUIDOrBadRequest(w, chi.URLParam(r, "skillId"), "skill_id")
if !ok {
return
}
var req struct {
Enabled *bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Enabled == nil {
writeError(w, http.StatusBadRequest, "enabled is required")
return
}
rows, err := h.Queries.SetAgentSkillEnabled(r.Context(), db.SetAgentSkillEnabledParams{
AgentID: agent.ID,
SkillID: skillID,
Enabled: *req.Enabled,
})
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to update agent skill")
return
}
if rows == 0 {
writeError(w, http.StatusNotFound, "agent skill not found")
return
}
h.writeUpdatedAgentSkills(w, r, agent)
}
func (h *Handler) RemoveAgentSkill(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
agent, ok := h.loadAgentForUser(w, r, id)
if !ok {
return
}
if !h.canManageAgent(w, r, agent) {
return
}
skillID, ok := parseUUIDOrBadRequest(w, chi.URLParam(r, "skillId"), "skill_id")
if !ok {
return
}
if err := h.Queries.RemoveAgentSkill(r.Context(), db.RemoveAgentSkillParams{
AgentID: agent.ID,
SkillID: skillID,
}); err != nil {
writeError(w, http.StatusInternalServerError, "failed to remove agent skill")
return
}
h.writeUpdatedAgentSkills(w, r, agent)
}
func (h *Handler) validateAgentSkillIDsInWorkspace(w http.ResponseWriter, r *http.Request, agent db.Agent, skillUUIDs []pgtype.UUID) bool {
seen := map[string]struct{}{}
for _, skillID := range skillUUIDs {
key := uuidToString(skillID)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
if _, err := h.Queries.GetSkillInWorkspace(r.Context(), db.GetSkillInWorkspaceParams{
ID: skillID,
WorkspaceID: agent.WorkspaceID,
}); err != nil {
writeError(w, http.StatusNotFound, "skill not found")
return false
}
}
return true
}
func (h *Handler) writeUpdatedAgentSkills(w http.ResponseWriter, r *http.Request, agent db.Agent) {
skills, err := h.Queries.ListAgentSkillSummaries(r.Context(), agent.ID)
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to list agent skills")
return
}
resp := make([]SkillSummaryResponse, len(skills))
for i, s := range skills {
resp[i] = skillSummaryToResponse(
s.ID, s.WorkspaceID, s.Name, s.Description, s.Config,
s.CreatedBy, s.CreatedAt, s.UpdatedAt,
)
resp[i].Enabled = &s.Enabled
}
actorType, actorID := h.resolveActor(r, requestUserID(r), uuidToString(agent.WorkspaceID))
h.publish(protocol.EventAgentStatus, uuidToString(agent.WorkspaceID), actorType, actorID, map[string]any{"agent_id": uuidToString(agent.ID), "skills": resp})
writeJSON(w, http.StatusOK, resp)
}