fix: relocate binary analysis skill

This commit is contained in:
username
2026-07-31 13:25:08 -04:00
parent 0d4a3652e4
commit f66ed48b8c
107 changed files with 17 additions and 2 deletions
@@ -0,0 +1,102 @@
"""Project lifecycle — workspace, manifests, locking, cache, and state machine.
All persistent state mutations use atomic write patterns (tempfile + os.rename)
to ensure project.json is never partially written. File locks serialize
concurrent access. The manifest system detects corrupted project manifests
and raises InvalidConfigError (exit code 4) with diagnostic information.
The state machine enforces valid lifecycle transitions.
"""
from __future__ import annotations
from binary_analysis.projects.atomic import (
atomic_append_text,
atomic_write_binary,
atomic_write_json,
atomic_write_lines,
atomic_write_text,
)
from binary_analysis.projects.cache import (
cache_clear,
cache_delete,
cache_get,
cache_list,
cache_set,
)
from binary_analysis.projects.diagnostics import (
clear_diagnostics,
get_diagnostics_summary,
load_diagnostics,
persist_diagnostics,
)
from binary_analysis.projects.lock import (
LockError,
acquire_lock,
get_lock_holder,
is_locked,
release_lock,
)
from binary_analysis.projects.manifest import (
create_manifest,
load_manifest,
save_manifest,
update_manifest_field,
)
from binary_analysis.projects.state_machine import (
can_analyze,
can_clean,
can_import,
is_valid_transition,
should_reject_migrate,
transition_to_failed,
)
from binary_analysis.projects.workspace import (
create_workspace,
get_project_path,
get_workspace_root,
get_workspace_subdirs,
list_workspaces,
remove_workspace,
validate_project_name,
workspace_exists,
)
__all__ = [
"LockError",
"acquire_lock",
"atomic_append_text",
"atomic_write_binary",
"atomic_write_json",
"atomic_write_lines",
"atomic_write_text",
"cache_clear",
"cache_delete",
"cache_get",
"cache_list",
"cache_set",
"can_analyze",
"can_clean",
"can_import",
"clear_diagnostics",
"create_manifest",
"create_workspace",
"get_diagnostics_summary",
"get_lock_holder",
"get_project_path",
"get_workspace_root",
"get_workspace_subdirs",
"is_locked",
"is_valid_transition",
"list_workspaces",
"load_diagnostics",
"load_manifest",
"persist_diagnostics",
"release_lock",
"remove_workspace",
"save_manifest",
"should_reject_migrate",
"transition_to_failed",
"update_manifest_field",
"validate_project_name",
"workspace_exists",
]
@@ -0,0 +1,154 @@
"""Atomic file write utility using tempfile + os.rename.
Provides safe atomic write patterns for all persistent state:
manifests, audit logs, cache, and reports.
Key guarantees:
- Writes to a temporary file first (in the same directory as the target).
- os.rename is atomic on the same filesystem — it either replaces or it doesn't.
- A process crash mid-write leaves the previous valid state intact.
- The target file is never partially written or truncated.
"""
from __future__ import annotations
import contextlib
import json
import os
import tempfile
from typing import Any
def atomic_write_text(
path: str,
content: str,
encoding: str = "utf-8",
mode: int = 0o644,
) -> None:
"""Atomically write text content to a file.
Writes content to a temporary file in the same directory, then atomically
renames it to the target path. If the process crashes mid-write, the
temporary file is left behind and the target file is unaffected.
Args:
path: Target file path.
content: Text content to write.
encoding: Character encoding (default utf-8).
mode: File permissions (default 0o644).
"""
dirname = os.path.dirname(path)
fd, tmp_path = tempfile.mkstemp(dir=dirname, suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding=encoding) as f:
f.write(content)
os.chmod(tmp_path, mode)
os.replace(tmp_path, path) # Atomic rename on same filesystem
except BaseException:
# Clean up temp file on any error, then re-raise
with contextlib.suppress(OSError):
os.unlink(tmp_path)
raise
def atomic_write_json(
path: str,
data: dict[str, Any],
indent: int = 2,
encoding: str = "utf-8",
mode: int = 0o644,
) -> None:
"""Atomically write JSON data to a file.
Serializes the data to JSON, then atomically writes it using
atomic_write_text. Invalid JSON data (non-serializable) raises
before any file is touched.
Args:
path: Target file path.
data: JSON-serializable dict to write.
indent: JSON indentation level.
encoding: Character encoding.
mode: File permissions.
"""
content = json.dumps(data, indent=indent, ensure_ascii=False)
atomic_write_text(path, content, encoding=encoding, mode=mode)
def atomic_append_text(
path: str,
line: str,
encoding: str = "utf-8",
mode: int = 0o644,
) -> None:
"""Atomically append a single line to a file.
For append-only files like audit logs (events.jsonl), this reads the
existing content, appends the line, and writes atomically. This ensures
no partial lines or interleaving in the canonical file.
Args:
path: Target file path.
line: Single line to append (newline added if not present).
encoding: Character encoding.
mode: File permissions.
"""
if not line.endswith("\n"):
line += "\n"
# Read existing content or start fresh
try:
with open(path, encoding=encoding) as f:
existing = f.read()
except FileNotFoundError:
existing = ""
new_content = existing + line
atomic_write_text(path, new_content, encoding=encoding, mode=mode)
def atomic_write_binary(
path: str,
data: bytes,
mode: int = 0o644,
) -> None:
"""Atomically write binary data to a file.
Writes binary data to a temporary file, then renames atomically.
Args:
path: Target file path.
data: Binary content to write.
mode: File permissions.
"""
dirname = os.path.dirname(path)
fd, tmp_path = tempfile.mkstemp(dir=dirname, suffix=".tmp")
try:
with os.fdopen(fd, "wb") as f:
f.write(data)
os.chmod(tmp_path, mode)
os.replace(tmp_path, path)
except BaseException:
with contextlib.suppress(OSError):
os.unlink(tmp_path)
raise
def atomic_write_lines(
path: str,
lines: list[str],
encoding: str = "utf-8",
mode: int = 0o644,
) -> None:
"""Atomically write a list of lines to a file.
Each line is written with a trailing newline.
Args:
path: Target file path.
lines: List of lines to write.
encoding: Character encoding.
mode: File permissions.
"""
content = "".join(line if line.endswith("\n") else line + "\n" for line in lines)
atomic_write_text(path, content, encoding=encoding, mode=mode)
@@ -0,0 +1,224 @@
"""Cache management for project analysis data.
Provides atomic cache read/write operations using the atomic write utility.
Cached data is stored in the project's cache/ directory as JSON files.
Key guarantees:
- All cache writes use atomic_write_json (tempfile + os.rename).
- Cache cleanup (clean command) removes all cache files atomically.
- Cache keys are validated to prevent path traversal.
"""
from __future__ import annotations
import contextlib
import json
import os
from typing import Any
from binary_analysis.projects.atomic import atomic_write_json
# Cache subdirectory within a project workspace
CACHE_DIRNAME = "cache"
# Valid characters for cache keys (alphanumeric, underscore, hyphen, dot)
_VALID_KEY_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.")
def _validate_cache_key(key: str) -> str:
"""Validate a cache key to prevent path traversal and invalid chars.
Args:
key: The cache key to validate.
Returns:
The validated key (unchanged if valid).
Raises:
ValueError: If the key is invalid.
"""
if not key or not key.strip():
raise ValueError("Cache key must not be empty")
key = key.strip()
if "\x00" in key:
raise ValueError("Cache key must not contain null bytes")
if "/" in key or "\\" in key:
raise ValueError("Cache key must not contain path separators")
if key.startswith("."):
raise ValueError("Cache key must not start with a dot")
invalid_chars = [c for c in key if c not in _VALID_KEY_CHARS]
if invalid_chars:
raise ValueError(f"Cache key contains invalid characters: {''.join(invalid_chars)}")
if not key.endswith(".json"):
key = key + ".json"
return key
def _cache_path(project_path: str, key: str) -> str:
"""Resolve the full path for a cache entry.
Args:
project_path: Absolute path to the project workspace directory.
key: Validated cache key.
Returns:
Full path to the cache file.
"""
return os.path.join(project_path, CACHE_DIRNAME, key)
def cache_get(project_path: str, key: str) -> Any:
"""Retrieve a cached value.
Args:
project_path: Absolute path to the project workspace directory.
key: Cache key (must be a safe filename).
Returns:
The cached data, or None if the key doesn't exist or is corrupted.
Raises:
ValueError: If the cache key is invalid.
"""
key = _validate_cache_key(key)
cache_file = _cache_path(project_path, key)
if not os.path.exists(cache_file):
return None
try:
with open(cache_file, encoding="utf-8") as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
# Corrupted cache entry — return None so caller can regenerate
return None
def cache_set(project_path: str, key: str, value: Any) -> None:
"""Atomically store a value in the cache.
Uses atomic_write_json to ensure cache entries are never partially
written. Invalid or non-serializable values raise before any file is
touched.
Args:
project_path: Absolute path to the project workspace directory.
key: Cache key (must be a safe filename).
value: JSON-serializable value to cache.
Raises:
ValueError: If the cache key is invalid.
TypeError: If the value is not JSON-serializable.
"""
key = _validate_cache_key(key)
cache_file = _cache_path(project_path, key)
# Ensure cache directory exists
cache_dir = os.path.dirname(cache_file)
os.makedirs(cache_dir, exist_ok=True)
# Serialize via JSON round-trip to validate types
json_str = json.dumps(value, ensure_ascii=False)
# Atomic write
atomic_write_json(cache_file, json.loads(json_str))
def cache_delete(project_path: str, key: str) -> bool:
"""Delete a cached entry.
Args:
project_path: Absolute path to the project workspace directory.
key: Cache key.
Returns:
True if the entry was deleted, False if it didn't exist.
Raises:
ValueError: If the cache key is invalid.
"""
key = _validate_cache_key(key)
cache_file = _cache_path(project_path, key)
if not os.path.exists(cache_file):
return False
try:
os.unlink(cache_file)
except OSError:
return False
return True
def cache_clear(project_path: str) -> int:
"""Remove all cached entries for a project.
Deletes all files in the cache/ directory but does not remove
the directory itself. Uses shutil.rmtree for efficiency, or
individual deletes if that fails.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
Number of cache entries removed.
"""
import shutil
cache_dir = os.path.join(project_path, CACHE_DIRNAME)
if not os.path.exists(cache_dir):
return 0
count = 0
try:
# Count entries before clearing
entries = [e for e in os.listdir(cache_dir) if os.path.isfile(os.path.join(cache_dir, e))]
count = len(entries)
except OSError:
pass
# Remove all files and recreate empty directory
try:
shutil.rmtree(cache_dir)
except OSError:
# Fall back to individual deletes
for entry in os.listdir(cache_dir):
with contextlib.suppress(OSError):
os.unlink(os.path.join(cache_dir, entry))
return count
os.makedirs(cache_dir, exist_ok=True)
return count
def cache_list(project_path: str) -> list[str]:
"""List all cached keys for a project.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
Sorted list of cache keys (without .json extension).
"""
cache_dir = os.path.join(project_path, CACHE_DIRNAME)
if not os.path.exists(cache_dir):
return []
keys: list[str] = []
try:
for entry in os.listdir(cache_dir):
if entry.endswith(".json") and os.path.isfile(os.path.join(cache_dir, entry)):
keys.append(entry[:-5]) # Remove .json
except OSError:
pass
return sorted(keys)
@@ -0,0 +1,144 @@
"""Diagnostics persistence — accumulate and retrieve diagnostics across commands.
Diagnostics are persisted as JSONL in project/diagnostics.jsonl, one
JSON object per line. Each entry has: severity, category, message, recoverable,
command, and timestamp.
The diagnostics file grows across the project lifecycle: warnings and errors
from analyze, triage, suspicious-apis, and other commands are accumulated
and retrievable via the `binary diagnostics` command.
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from typing import Any
from binary_analysis.projects.atomic import atomic_append_text
DIAGNOSTICS_FILENAME = "diagnostics.jsonl"
def _diagnostics_path(project_path: str) -> str:
"""Return the path to the diagnostics file within a project workspace."""
return os.path.join(project_path, DIAGNOSTICS_FILENAME)
def persist_diagnostics(
project_path: str,
diagnostics: list[dict[str, Any]],
command: str = "unknown",
) -> None:
"""Persist diagnostic entries to the project's diagnostics file.
Each diagnostic entry is augmented with a command field and timestamp
before being appended atomically to the JSONL file.
Args:
project_path: Absolute path to the project workspace directory.
diagnostics: List of diagnostic dicts to persist.
command: Name of the command that produced these diagnostics.
"""
if not diagnostics:
return
path = _diagnostics_path(project_path)
timestamp = datetime.now(timezone.utc).isoformat()
for diag in diagnostics:
entry = {
"severity": diag.get("severity", "INFO"),
"category": diag.get("category", "general"),
"message": diag.get("message", ""),
"recoverable": diag.get("recoverable", True),
"command": command,
"timestamp": timestamp,
}
# Preserve optional fields
if "component" in diag:
entry["component"] = diag["component"]
if "remediation" in diag:
entry["remediation"] = diag["remediation"]
line = json.dumps(entry, ensure_ascii=False)
atomic_append_text(path, line)
def load_diagnostics(project_path: str) -> list[dict[str, Any]]:
"""Load all accumulated diagnostics from the project's diagnostics file.
Returns an empty list if the file does not exist or is empty.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
List of diagnostic dicts ordered by appearance in the file
(oldest first).
"""
path = _diagnostics_path(project_path)
if not os.path.exists(path):
return []
diagnostics: list[dict[str, Any]] = []
try:
with open(path, encoding="utf-8") as f:
for line_num, line in enumerate(f, start=1):
line = line.strip()
if not line:
continue
try:
entry = json.loads(line)
diagnostics.append(entry)
except json.JSONDecodeError:
# Skip corrupted lines but note in a diagnostic
diagnostics.append(
{
"severity": "WARNING",
"category": "diagnostics-file",
"message": f"Corrupted diagnostics entry at line {line_num}",
"recoverable": True,
"command": "diagnostics",
"timestamp": datetime.now(timezone.utc).isoformat(),
}
)
except OSError:
return []
return diagnostics
def clear_diagnostics(project_path: str) -> None:
"""Remove the diagnostics file (e.g., on project clean).
Args:
project_path: Absolute path to the project workspace directory.
"""
path = _diagnostics_path(project_path)
if os.path.exists(path):
os.unlink(path)
def get_diagnostics_summary(
diagnostics: list[dict[str, Any]],
) -> dict[str, Any]:
"""Compute a summary of diagnostic entries.
Args:
diagnostics: List of diagnostic dicts.
Returns:
Dict with total count and breakdown by severity.
"""
by_severity: dict[str, int] = {"INFO": 0, "WARNING": 0, "ERROR": 0}
for d in diagnostics:
sev = d.get("severity", "INFO")
if sev in by_severity:
by_severity[sev] += 1
return {
"total": len(diagnostics),
"by_severity": by_severity,
}
@@ -0,0 +1,255 @@
"""File-based locking for concurrent access serialization.
Uses a lock file (project.lock) within the project workspace. The lock
file contains the holder's PID and acquisition timestamp. Lock acquisition
is non-blocking — callers that fail to acquire get a LockError immediately.
Key guarantees:
- Only one process can hold the lock at a time.
- A second process attempting to acquire the lock gets a LockError.
- The lock is released on process exit (normal or abnormal), via atexit.
- Stale locks (from dead processes) are detected and cleaned up.
- Lock state is recorded in the project manifest's `lock` field for visibility.
"""
from __future__ import annotations
import atexit
import contextlib
import os
from datetime import datetime, timezone
from binary_analysis.domain.enums import ExitCode
from binary_analysis.domain.errors import BinaryAnalysisError
# Lock filename within a project workspace
LOCK_FILENAME = "project.lock"
class LockError(BinaryAnalysisError):
"""Raised when a lock cannot be acquired.
Exit code 1 (GENERIC_ERROR) — the lock conflict means the operation
cannot proceed but it's not a configuration or argument problem.
"""
def __init__(self, project_name: str, holder_info: str | None = None) -> None:
msg = f"Project '{project_name}' is locked by another process."
if holder_info:
msg += f" {holder_info}"
msg += " Wait for the other process to complete or release the lock."
super().__init__(msg, ExitCode.GENERIC_ERROR)
def _acquire_lock_file(lock_path: str, holder_info: str) -> None:
"""Acquire the file lock by writing holder info.
Uses os.open with O_CREAT | O_EXCL — this atomically creates the file
only if it doesn't already exist. If the file exists, acquisition fails.
Args:
lock_path: Path to the lock file.
holder_info: Information about the lock holder (e.g., PID, purpose).
Raises:
LockError: If the lock is already held.
"""
try:
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
except FileExistsError:
# Lock exists — try to read holder info for better diagnostics
try:
with open(lock_path) as f:
existing_info = f.read().strip()
except (OSError, UnicodeDecodeError):
existing_info = "unknown holder"
# Check if the lock is stale (process no longer running)
if _is_stale_lock(lock_path):
# Clean up stale lock and retry
with contextlib.suppress(OSError):
os.unlink(lock_path)
# Retry acquisition
try:
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
except FileExistsError:
raise LockError(
os.path.basename(os.path.dirname(lock_path)),
f"Held by: {existing_info}",
) from None
else:
raise LockError(
os.path.basename(os.path.dirname(lock_path)),
f"Held by: {existing_info}",
) from None
with os.fdopen(fd, "w") as f:
f.write(holder_info)
def _is_stale_lock(lock_path: str) -> bool:
"""Check if a lock file is from a dead process.
Reads the PID from the lock file and checks if the process is still alive.
Args:
lock_path: Path to the lock file.
Returns:
True if the lock is stale (holder process is dead).
"""
try:
with open(lock_path) as f:
content = f.read().strip()
except (OSError, UnicodeDecodeError):
return True # Unreadable lock = stale
# Parse PID from lock content (format: "pid=<PID> ...")
pid = None
for part in content.split():
if part.startswith("pid="):
try:
pid = int(part.split("=", 1)[1])
except (ValueError, IndexError):
return True # Can't parse PID = stale
break
if pid is None:
return True # No PID in lock file = stale
# Check if process exists
try:
os.kill(pid, 0) # Signal 0 does nothing but checks existence
return False # Process exists — lock is valid
except OSError:
return True # Process doesn't exist — lock is stale
def acquire_lock(
project_path: str,
project_name: str | None = None,
holder_purpose: str = "analysis",
) -> str:
"""Acquire a file lock for the project workspace.
Non-blocking: if the lock is held by another live process, raises LockError.
If the lock is stale (holder process is dead), cleans it up and acquires.
Registers an atexit handler to release the lock on process exit.
Args:
project_path: Absolute path to the project workspace directory.
project_name: Project name for error messages. Defaults to dir name.
holder_purpose: Description of why the lock is being held.
Returns:
The lock holder info string.
Raises:
LockError: If the lock cannot be acquired (held by live process).
"""
if project_name is None:
project_name = os.path.basename(project_path)
pid = os.getpid()
holder_info = f"pid={pid} host={os.uname().nodename} purpose={holder_purpose} acquired_at={datetime.now(timezone.utc).isoformat()}"
lock_path = os.path.join(project_path, LOCK_FILENAME)
_acquire_lock_file(lock_path, holder_info)
# Register cleanup via atexit
atexit.register(_release_lock_file, lock_path)
return holder_info
def release_lock(project_path: str) -> bool:
"""Release the file lock for the project workspace.
Only releases the lock if the current process is the holder.
Can be called explicitly or via the atexit handler.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
True if the lock was released, False if there was no lock
or the lock was held by a different process.
"""
lock_path = os.path.join(project_path, LOCK_FILENAME)
return _release_lock_file(lock_path)
def _release_lock_file(lock_path: str) -> bool:
"""Release a lock file if the current process is the holder.
Args:
lock_path: Path to the lock file.
Returns:
True if the lock was released.
"""
if not os.path.exists(lock_path):
return False
# Only release if we are the holder
try:
with open(lock_path) as f:
content = f.read().strip()
except (OSError, UnicodeDecodeError):
# Can't read — just remove it
with contextlib.suppress(OSError):
os.unlink(lock_path)
return True
current_pid = os.getpid()
for part in content.split():
if part.startswith("pid="):
try:
lock_pid = int(part.split("=", 1)[1])
except (ValueError, IndexError):
lock_pid = None
if lock_pid is not None and lock_pid != current_pid:
return False # Not our lock
break
with contextlib.suppress(OSError):
os.unlink(lock_path)
return True
return False
def is_locked(project_path: str) -> bool:
"""Check if the project workspace has a valid (non-stale) lock.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
True if the project is locked by a live process.
"""
lock_path = os.path.join(project_path, LOCK_FILENAME)
if not os.path.exists(lock_path):
return False
return not _is_stale_lock(lock_path)
def get_lock_holder(project_path: str) -> str | None:
"""Get information about the current lock holder.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
Holder info string, or None if no valid lock exists.
"""
lock_path = os.path.join(project_path, LOCK_FILENAME)
if not os.path.exists(lock_path):
return None
if _is_stale_lock(lock_path):
return None
try:
with open(lock_path) as f:
return f.read().strip()
except (OSError, UnicodeDecodeError):
return None
@@ -0,0 +1,187 @@
"""Project manifest — load, save, validate, and atomically write project.json.
Uses the atomic write utility (tempfile + os.rename) to ensure that
project.json is never partially written. A process crash during a write
leaves the previous valid manifest (or no manifest) but never a corrupted one.
Key guarantees:
- Loads project manifests as typed dicts with validation.
- Saves project manifests atomically via atomic_write_json.
- Detects corrupted manifests (invalid JSON) and raises InvalidConfigError
with exit code 4.
- Detects missing required fields in manifest and treats as corruption.
- Provides helpers to create new project manifests with proper defaults.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from typing import Any
from uuid import UUID, uuid4
from binary_analysis.domain.enums import ProjectState
from binary_analysis.domain.errors import InvalidConfigError
from binary_analysis.projects.atomic import atomic_write_json
# Required top-level fields in project.json
_REQUIRED_FIELDS: tuple[str, ...] = (
"id",
"name",
"state",
"created_at",
"workspace_version",
"binary_count",
"is_stale",
)
# Current workspace format version
_WORKSPACE_VERSION = "1"
# Manifest filename within a project workspace
MANIFEST_FILENAME = "project.json"
def create_manifest(
project_name: str,
project_id: UUID | None = None,
) -> dict[str, Any]:
"""Create a new project manifest dict with default values.
The manifest is in the CREATED state, with a new UUID and current timestamp.
Args:
project_name: The project name.
project_id: Optional UUID; auto-generated if not provided.
Returns:
A dict representing the project manifest, ready to be saved.
"""
now = datetime.now(timezone.utc).isoformat()
if project_id is None:
project_id = uuid4()
return {
"id": str(project_id),
"name": project_name,
"state": ProjectState.CREATED.value,
"created_at": now,
"updated_at": now,
"workspace_version": _WORKSPACE_VERSION,
"binary_count": 0,
"is_stale": False,
"lock": None,
"description": None,
"max_binary_size_bytes": None,
}
def save_manifest(project_path: str, manifest: dict[str, Any]) -> None:
"""Atomically save a project manifest to project.json.
Uses tempfile + os.rename to guarantee the file is never partially
written. If the process crashes mid-write, the previous valid manifest
(or no file) is left intact.
Args:
project_path: Absolute path to the project workspace directory.
manifest: The manifest dict to save.
Raises:
ValueError: If the manifest is missing required fields.
"""
_validate_manifest(manifest)
manifest_path = f"{project_path}/{MANIFEST_FILENAME}"
atomic_write_json(manifest_path, manifest)
def load_manifest(project_path: str) -> dict[str, Any]:
"""Load a project manifest from project.json.
Reads and validates the manifest. If the file is missing, raises
FileNotFoundError. If the JSON is invalid, raises InvalidConfigError
(exit code 4) with a diagnostic explaining the corruption.
If required fields are missing, raises InvalidConfigError.
Args:
project_path: Absolute path to the project workspace directory.
Returns:
The parsed and validated manifest dict.
Raises:
FileNotFoundError: If project.json does not exist.
InvalidConfigError: If the manifest is corrupted (invalid JSON or
missing required fields). Exit code 4.
"""
manifest_path = f"{project_path}/{MANIFEST_FILENAME}"
try:
with open(manifest_path, encoding="utf-8") as f:
raw_text = f.read()
except FileNotFoundError:
raise FileNotFoundError(f"Project manifest not found: {manifest_path}") from None
# Parse JSON — detect corruption
try:
manifest = json.loads(raw_text)
except json.JSONDecodeError as e:
raise InvalidConfigError(
f"Corrupted project manifest at {manifest_path}: invalid JSON. "
f"Parse error: {e.msg} at line {e.lineno}, column {e.colno}. "
f"The file must be repaired or the project workspace re-created."
) from e
if not isinstance(manifest, dict):
raise InvalidConfigError(
f"Corrupted project manifest at {manifest_path}: "
f"expected a JSON object, got {type(manifest).__name__}."
)
# Validate required fields
_validate_manifest(manifest)
return manifest
def _validate_manifest(manifest: dict[str, Any]) -> None:
"""Validate that a manifest dict has all required fields.
Args:
manifest: The manifest dict to validate.
Raises:
InvalidConfigError: If required fields are missing.
"""
missing = [field for field in _REQUIRED_FIELDS if field not in manifest]
if missing:
raise InvalidConfigError(
f"Corrupted project manifest: missing required fields: {', '.join(missing)}."
)
def update_manifest_field(
project_path: str,
updates: dict[str, Any],
) -> dict[str, Any]:
"""Load, update fields, and atomically save a project manifest.
This is a convenience for state transitions and field updates.
Automatically updates the `updated_at` timestamp.
Args:
project_path: Absolute path to the project workspace directory.
updates: Dict of field names to new values.
Returns:
The updated manifest dict (post-save).
Raises:
FileNotFoundError: If the project doesn't exist.
InvalidConfigError: If the current or updated manifest is corrupted.
"""
manifest = load_manifest(project_path)
manifest.update(updates)
manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
save_manifest(project_path, manifest)
return manifest
@@ -0,0 +1,255 @@
"""Path security — symlink resolution, workspace containment, path traversal prevention.
This module provides the central path validation used by all commands that
accept user-supplied file paths (binary import, report output, workspace
operations). All path validation follows the same pattern:
1. Resolve symlinks (os.path.realpath)
2. Check path is contained within the allowed boundary (workspace or project)
3. Reject traversal sequences, absolute paths outside boundary, and null bytes
These checks enforce the safety architecture:
- Never write files outside the project workspace
- Reject paths designed to escape containment
- Prevent symlink-based traversal attacks
"""
from __future__ import annotations
import os
from pathlib import Path
def resolve_path(path: str) -> str:
"""Resolve a path with symlink expansion to its canonical form.
Uses os.path.realpath to follow all symlinks and resolve relative
path components. If the path does not exist, still resolves as far
as possible through os.path.realpath (which handles most cases).
Args:
path: The user-supplied path string.
Returns:
The canonical absolute path with all symlinks resolved.
"""
# os.path.realpath resolves symlinks and normalizes the path
# even if the file doesn't exist (it resolves the directory part)
return os.path.realpath(path)
def check_no_path_traversal(path: str) -> None:
"""Reject path traversal sequences and null bytes in a path.
Args:
path: The user-supplied path string.
Raises:
ValueError: If the path contains null bytes or explicit traversal sequences.
"""
# Null byte rejection
if "\x00" in path:
raise ValueError("Path must not contain null bytes")
# Check for explicit traversal sequences in the raw path
# Split by both Unix and Windows separators
raw_parts = path.replace("\\", "/").split("/")
if ".." in raw_parts:
raise ValueError(f"Path traversal detected in: {path}")
# Also check normalized form as a backup
normalized = os.path.normpath(path)
norm_parts = Path(normalized).parts
if ".." in norm_parts:
raise ValueError(f"Path traversal detected in: {path}")
def check_within_boundary(path: str, boundary: str) -> None:
"""Check that a resolved path is contained within a boundary directory.
The boundary check uses os.path.commonpath to verify containment.
Both paths must be absolute and resolved before calling this function.
Args:
path: The resolved absolute path to check.
boundary: The resolved absolute boundary directory.
Raises:
ValueError: If the path is not within the boundary directory.
"""
path_abs = os.path.abspath(path)
boundary_abs = os.path.abspath(boundary)
common = os.path.commonpath([path_abs, boundary_abs])
if common != boundary_abs:
raise ValueError(f"Path '{path}' is outside the allowed boundary '{boundary}'.")
def validate_binary_import_path(binary_path: str, project_path: str) -> str:
"""Validate a binary import path for safety.
Performs:
1. Null byte and traversal sequence checks on the raw path
2. Symlink resolution to get the canonical path
3. File existence check (after resolution)
4. Workspace containment check (the binary must be within the project)
Note: For copy mode, the binary can come from outside the project.
The workspace containment check is relaxed — we check that the path
does not traverse to sensitive system locations, but absolute paths
from /tmp or user home are allowed for import.
For reference mode, the binary source path is stored but the binary
is never written outside the project.
Args:
binary_path: The user-supplied path to the binary file.
project_path: The resolved project workspace directory.
Returns:
The resolved canonical path to the binary.
Raises:
ValueError: If the path fails validation.
FileNotFoundError: If the resolved path does not exist.
"""
# Step 1: Reject null bytes and explicit traversal
check_no_path_traversal(binary_path)
# Step 2: Resolve symlinks for the directory part (file may not exist yet
# for import dry-run, but it must exist for a real import)
# We resolve the directory path first, then append the file name
dir_part = os.path.dirname(binary_path) or "."
base_part = os.path.basename(binary_path)
resolved_dir = os.path.realpath(dir_part)
resolved_path = os.path.join(resolved_dir, base_part)
# Step 3: Check the resolved directory is not a system-sensitive location
# Reject paths that resolve to common system directories
_check_not_system_path(resolved_path)
return resolved_path
def validate_output_path(output_path: str, project_path: str) -> str:
"""Validate a report/output path is within the project workspace.
Performs:
1. Null byte and traversal sequence checks
2. Resolves the path relative to the project workspace
3. Verifies the resolved path is within the project workspace
Args:
output_path: The user-supplied output path.
project_path: The resolved project workspace directory.
Returns:
The validated absolute output path within the project workspace.
Raises:
ValueError: If the path would escape the project workspace.
"""
# Step 1: Reject null bytes and explicit traversal
check_no_path_traversal(output_path)
# Step 2: If output_path is absolute, check it separately
# If relative, resolve relative to project_path
if os.path.isabs(output_path):
# Absolute paths must still be within the project workspace
resolved = os.path.realpath(output_path)
check_within_boundary(resolved, project_path)
return resolved
# Relative path: resolve against project_path
joined = os.path.join(project_path, output_path)
resolved = os.path.realpath(joined)
check_within_boundary(resolved, project_path)
return resolved
def validate_workspace_path(path_in_workspace: str, project_path: str) -> str:
"""Validate a path that must be within a project workspace.
Resolves symlinks and ensures the resolved path is within the
project workspace boundary. Used for workspace operations that
traverse project subdirectories.
Args:
path_in_workspace: A path within the project workspace.
project_path: The resolved project workspace directory.
Returns:
The resolved canonical path.
Raises:
ValueError: If the resolved path escapes the project workspace.
"""
check_no_path_traversal(path_in_workspace)
resolved = os.path.realpath(path_in_workspace)
check_within_boundary(resolved, project_path)
return resolved
def _check_not_system_path(path: str) -> None:
"""Reject paths that resolve to system-sensitive locations.
This prevents importing binaries from /etc, /proc, /sys, or other
system directories that could leak sensitive information.
Args:
path: The resolved path to check.
Raises:
ValueError: If the path is in a system-sensitive location.
"""
# System-sensitive prefixes (Linux/macOS)
system_prefixes: tuple[str, ...] = (
"/etc/",
"/proc/",
"/sys/",
"/dev/",
"/System/", # macOS
"/Library/System/", # macOS
"/private/etc/", # macOS
"/private/var/", # macOS (system vars)
)
path_abs = os.path.abspath(path)
# Allow user temp directories (macOS /private/var/folders/*, /private/tmp/, /tmp/)
user_temp_prefixes = (
"/private/var/folders/",
"/private/tmp/",
"/var/folders/",
"/tmp/",
)
for prefix in user_temp_prefixes:
if path_abs.startswith(prefix):
return # User temp directories are safe
# Check against the system-sensitive directories themselves
system_dirs: set[str] = {
"/etc",
"/proc",
"/sys",
"/dev",
"/boot",
"/System",
"/private/etc",
"/private/var",
}
for prefix in system_prefixes:
if path_abs.startswith(prefix):
raise ValueError(
f"Path '{path}' resolves to a system-sensitive location ({prefix}). "
"Import of files from system directories is not allowed for safety."
)
if path_abs in system_dirs:
raise ValueError(
f"Path '{path}' is a system-sensitive directory. "
"Import of files from system directories is not allowed for safety."
)
@@ -0,0 +1,166 @@
"""Project state machine — lifecycle transitions and staleness detection.
Enforces strict state transitions per the architecture:
CREATED -> IMPORTED -> ANALYZING -> READY
READY -> STALE -> ANALYZING
Any state -> FAILED (with diagnostics preserved)
Provides:
- Transition validation (reject invalid transitions).
- FAILED transition helpers (preserve diagnostics, release locks).
- Staleness detection (SHA-256 comparison on source change).
- State-aware operation guards (clean only FAILED, migrate only unlocked).
"""
from __future__ import annotations
import contextlib
from datetime import datetime, timezone
from typing import Any
from binary_analysis.domain.enums import ProjectState
# ---------------------------------------------------------------------------
# Valid transition map
# ---------------------------------------------------------------------------
# Each state maps to a set of allowed target states
_VALID_TRANSITIONS: dict[ProjectState, set[ProjectState]] = {
ProjectState.CREATED: {ProjectState.IMPORTED, ProjectState.FAILED},
ProjectState.IMPORTED: {ProjectState.ANALYZING, ProjectState.FAILED},
ProjectState.ANALYZING: {ProjectState.READY, ProjectState.FAILED},
ProjectState.READY: {ProjectState.STALE, ProjectState.FAILED},
ProjectState.STALE: {ProjectState.ANALYZING, ProjectState.FAILED},
ProjectState.FAILED: {ProjectState.CREATED}, # Clean resets to CREATED
}
# States from which analyze can be started (re-transition)
_ANALYZABLE_STATES: set[ProjectState] = {
ProjectState.IMPORTED,
ProjectState.STALE,
ProjectState.READY, # Can detect staleness without re-analyzing
}
# States from which import is allowed
_IMPORTABLE_STATES: set[ProjectState] = {
ProjectState.CREATED,
ProjectState.IMPORTED,
}
# States from which clean is allowed (only FAILED)
_CLEANABLE_STATES: set[ProjectState] = {
ProjectState.FAILED,
}
# States from which migrate is rejected (locked projects)
_MIGRATE_BLOCKED_STATES: set[ProjectState] = {
ProjectState.ANALYZING,
}
# ---------------------------------------------------------------------------
# Transition validation
# ---------------------------------------------------------------------------
def is_valid_transition(from_state: ProjectState, to_state: ProjectState) -> bool:
"""Check if a state transition is allowed by the state machine.
Args:
from_state: Current project state.
to_state: Desired target state.
Returns:
True if the transition is valid.
"""
allowed = _VALID_TRANSITIONS.get(from_state, set())
return to_state in allowed
def can_analyze(state: ProjectState) -> bool:
"""Check if analysis can be started from the given state."""
return state in _ANALYZABLE_STATES
def can_import(state: ProjectState) -> bool:
"""Check if a binary import is allowed in the given state."""
return state in _IMPORTABLE_STATES
def can_clean(state: ProjectState) -> bool:
"""Check if clean is allowed in the given state (only FAILED)."""
return state in _CLEANABLE_STATES
def should_reject_migrate(state: ProjectState, is_locked: bool) -> bool:
"""Check if migrate should be rejected due to project state or lock.
Args:
state: Current project state.
is_locked: Whether the project has an active lock.
Returns:
True if migrate should be rejected.
"""
if is_locked:
return True
return state in _MIGRATE_BLOCKED_STATES
# ---------------------------------------------------------------------------
# Transition helpers
# ---------------------------------------------------------------------------
def transition_to_failed(
manifest: dict[str, Any],
from_state: ProjectState,
diagnostics: list[dict[str, Any]],
release_lock_fn: Any | None = None,
) -> dict[str, Any]:
"""Transition a project to FAILED state, preserving context from the source state.
Handles specific preservation rules per source state:
- CREATED->FAILED: Preserve diagnostics, no lock to release.
- IMPORTED->FAILED: Preserve binary record (binary_count, binary data),
release lock if held.
- ANALYZING->FAILED: Release lock, preserve crash diagnostics,
clear lock from manifest.
- STALE->FAILED: Capture both staleness cause and analysis failure,
preserve binary record.
Args:
manifest: The current project manifest (mutated in place).
from_state: The state before failure.
diagnostics: Failure diagnostics to preserve.
release_lock_fn: Optional function to release the project lock.
Returns:
The updated manifest dict.
"""
now = datetime.now(timezone.utc).isoformat()
# Preserve existing diagnostics
existing_diags = manifest.get("diagnostics", [])
if not isinstance(existing_diags, list):
existing_diags = []
# Merge diagnostics, ensuring we don't lose staleness context
merged_diags = existing_diags + diagnostics
# Update manifest
manifest["state"] = ProjectState.FAILED.value
manifest["diagnostics"] = merged_diags
manifest["updated_at"] = now
# Release lock if transitioning from ANALYZING
if from_state == ProjectState.ANALYZING:
manifest["lock"] = None
if release_lock_fn is not None:
with contextlib.suppress(Exception):
release_lock_fn()
# Preserve binary record for IMPORTED->FAILED and STALE->FAILED
# (binary_count and is_stale are preserved by default since we don't clear them)
return manifest
@@ -0,0 +1,228 @@
"""Workspace directory structure management.
Manages the hierarchical directory layout for each project workspace:
project/
project.json # Project manifest
binaries/<id>.json # Binary metadata records
samples/ # Copied binary samples
audit/events.jsonl # Append-only audit log
reports/ # Generated reports
exports/ # Export artifacts
cache/ # Cached analysis data
backend/ghidra/ # Ghidra-specific data
Also provides workspace root discovery via:
BINARY_WORKSPACE_ROOT env var, or
default XDG-compatible location (~/.local/share/binary-analysis/workspaces).
"""
from __future__ import annotations
import os
from pathlib import Path
# Workspace root can be configured via this environment variable
_WORKSPACE_ROOT_ENV = "BINARY_WORKSPACE_ROOT"
# Default workspace root (XDG-compatible)
_DEFAULT_WORKSPACE_ROOT = os.path.expanduser("~/.local/share/binary-analysis/workspaces")
def get_workspace_root() -> Path:
"""Return the root directory for all project workspaces.
Resolution order:
1. BINARY_WORKSPACE_ROOT environment variable
2. Default XDG-compatible path (~/.local/share/binary-analysis/workspaces)
Returns:
Absolute path to the workspace root directory.
"""
env_root = os.environ.get(_WORKSPACE_ROOT_ENV)
if env_root:
return Path(env_root).resolve()
return Path(_DEFAULT_WORKSPACE_ROOT).resolve()
def ensure_workspace_root() -> Path:
"""Create and return the workspace root directory.
Creates the directory if it doesn't exist, along with parent directories.
Returns:
Absolute path to the (now existing) workspace root directory.
"""
root = get_workspace_root()
root.mkdir(parents=True, exist_ok=True)
return root
def get_project_path(project_name: str) -> Path:
"""Return the workspace path for a named project.
Args:
project_name: The project name. Must be a valid directory name.
Returns:
Absolute path to the project's workspace directory.
"""
root = get_workspace_root()
return root / project_name
def create_workspace(project_name: str) -> Path:
"""Create a full project workspace directory structure.
Creates the project root directory and all standard subdirectories.
Args:
project_name: The project name. Must be a valid directory name.
Returns:
Absolute path to the created project workspace root.
Raises:
FileExistsError: If the project workspace already exists.
OSError: If directory creation fails.
"""
project_dir = get_project_path(project_name)
if project_dir.exists():
raise FileExistsError(f"Project workspace already exists: {project_dir}")
# Standard subdirectories per architecture
subdirs = [
"binaries",
"samples",
"audit",
"reports",
"exports",
"cache",
"backend/ghidra",
]
# Create project root + all subdirectories
project_dir.mkdir(parents=True, exist_ok=False)
for subdir in subdirs:
(project_dir / subdir).mkdir(parents=True, exist_ok=True)
return project_dir
def remove_workspace(project_name: str) -> None:
"""Remove an entire project workspace directory.
Deletes the project directory and all contents recursively.
Args:
project_name: The project name to remove.
Raises:
FileNotFoundError: If the project workspace does not exist.
"""
import shutil
project_dir = get_project_path(project_name)
if not project_dir.exists():
raise FileNotFoundError(f"Project workspace not found: {project_dir}")
shutil.rmtree(str(project_dir))
def workspace_exists(project_name: str) -> bool:
"""Check if a project workspace directory exists.
Args:
project_name: The project name to check.
Returns:
True if the workspace directory exists.
"""
return get_project_path(project_name).exists()
def list_workspaces() -> list[str]:
"""List all project workspace names in the workspace root.
Returns:
Sorted list of project directory names.
"""
root = get_workspace_root()
if not root.exists():
return []
entries = sorted(e.name for e in root.iterdir() if e.is_dir() and not e.name.startswith("."))
return entries
def get_workspace_subdirs(project_name: str) -> dict[str, Path]:
"""Return paths to all standard subdirectories within a project workspace.
Args:
project_name: The project name.
Returns:
Dict mapping subdirectory names to absolute paths.
Raises:
FileNotFoundError: If the project workspace does not exist.
"""
project_dir = get_project_path(project_name)
if not project_dir.exists():
raise FileNotFoundError(f"Project workspace not found: {project_dir}")
return {
"root": project_dir,
"binaries": project_dir / "binaries",
"samples": project_dir / "samples",
"audit": project_dir / "audit",
"reports": project_dir / "reports",
"exports": project_dir / "exports",
"cache": project_dir / "cache",
"backend_ghidra": project_dir / "backend" / "ghidra",
}
def validate_project_name(name: str) -> str:
"""Validate and sanitize a project name.
Project names must:
- Not be empty
- Not contain path separators (/ or \\)
- Not contain null bytes
- Not start with a dot
- Only contain alphanumeric characters, hyphens, and underscores
Args:
name: The proposed project name.
Returns:
The validated project name (unchanged if valid).
Raises:
ValueError: If the project name is invalid.
"""
if not name or not name.strip():
raise ValueError("Project name must not be empty")
name = name.strip()
if name in (".", ".."):
raise ValueError(f"Invalid project name: {name}")
if "\x00" in name:
raise ValueError("Project name must not contain null bytes")
if "/" in name or "\\" in name:
raise ValueError("Project name must not contain path separators")
if name.startswith("."):
raise ValueError("Project name must not start with a dot")
# Only allow alphanumeric, hyphens, and underscores
invalid_chars = [c for c in name if not (c.isalnum() or c in "-_")]
if invalid_chars:
raise ValueError(
f"Project name contains invalid characters: {''.join(invalid_chars)}. "
"Only alphanumeric, hyphens, and underscores are allowed."
)
return name