mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
741 lines
30 KiB
Python
741 lines
30 KiB
Python
"""Tests for binary import, analyze, and metadata CLI commands.
|
|
|
|
Validates all VAL-IMP assertions:
|
|
- Import: copy mode, reference mode, SHA-256 client-side, unsupported format (exit 5),
|
|
max size rejection, PROJECT_NOT_FOUND (exit 6), import during active analysis rejection
|
|
- Analyze: state transitions, lock lifecycle, profiles, timeout (exit 12),
|
|
staleness detection, unknown profile, BINARY_NOT_FOUND (exit 7),
|
|
hard analysis failure (exit 11), backend failure (exit 13)
|
|
- Metadata: canonical fields, project_state in provenance, backend-neutral output
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
_scripts_dir = Path(__file__).resolve().parents[2] / "scripts"
|
|
if str(_scripts_dir) not in sys.path:
|
|
sys.path.insert(0, str(_scripts_dir))
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from binary_analysis.cli.main import main
|
|
from binary_analysis.domain.enums import ExitCode, ProjectState
|
|
from binary_analysis.projects.lock import is_locked
|
|
from binary_analysis.projects.manifest import create_manifest, load_manifest, save_manifest
|
|
from binary_analysis.projects.workspace import (
|
|
create_workspace,
|
|
get_project_path,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def temp_workspace_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
|
"""Redirect workspace root to a temp directory for all tests."""
|
|
root = tmp_path / "workspaces"
|
|
root.mkdir(parents=True)
|
|
monkeypatch.setenv("BINARY_WORKSPACE_ROOT", str(root))
|
|
return root
|
|
|
|
|
|
@pytest.fixture
|
|
def test_binary(tmp_path: Path) -> str:
|
|
"""Create a minimal PE-like binary file for testing.
|
|
|
|
PE magic: 'MZ' at offset 0, 'PE\\0\\0' at offset after DOS stub.
|
|
Returns the path to the binary.
|
|
"""
|
|
binary_path = tmp_path / "test.exe"
|
|
# PE magic bytes: MZ header + PE signature at 0x80
|
|
content = bytearray(512)
|
|
content[0] = 0x4D # M
|
|
content[1] = 0x5A # Z
|
|
# PE signature at offset 0x80
|
|
content[0x80] = 0x50 # P
|
|
content[0x81] = 0x45 # E
|
|
content[0x82] = 0x00
|
|
content[0x83] = 0x00
|
|
binary_path.write_bytes(content)
|
|
return str(binary_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def small_binary(tmp_path: Path) -> str:
|
|
"""Create a tiny binary for max-size testing."""
|
|
binary_path = tmp_path / "tiny.bin"
|
|
binary_path.write_bytes(b"MZ\x00\x01" + b"\x00" * 60) # 64 bytes
|
|
return str(binary_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def large_binary(tmp_path: Path) -> str:
|
|
"""Create a larger binary for max-size testing."""
|
|
binary_path = tmp_path / "large.exe"
|
|
# ~16KB binary
|
|
content = bytearray(16384)
|
|
content[0] = 0x4D # M
|
|
content[1] = 0x5A # Z
|
|
content[0x80] = 0x50 # P
|
|
content[0x81] = 0x45 # E
|
|
content[0x82] = 0x00
|
|
content[0x83] = 0x00
|
|
binary_path.write_bytes(content)
|
|
return str(binary_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def unsupported_file(tmp_path: Path) -> str:
|
|
"""Create a plain text file (unsupported format)."""
|
|
path = tmp_path / "notes.txt"
|
|
path.write_text("This is just a text file, not a binary.", encoding="ascii")
|
|
return str(path)
|
|
|
|
|
|
def _capture_json(
|
|
args: list[str],
|
|
capsys: pytest.CaptureFixture,
|
|
stdin_text: str | None = None,
|
|
) -> tuple[int, dict]:
|
|
"""Run main() with --json and return (exit_code, parsed_json)."""
|
|
import sys as _sys
|
|
|
|
old_stdin = _sys.stdin
|
|
if stdin_text is not None:
|
|
_sys.stdin = io.StringIO(stdin_text)
|
|
try:
|
|
exit_code = main(["--json", *args])
|
|
finally:
|
|
_sys.stdin = old_stdin
|
|
captured = capsys.readouterr()
|
|
parsed = json.loads(captured.out) if captured.out.strip() else {}
|
|
return exit_code, parsed
|
|
|
|
|
|
def _make_created_project(name: str) -> str:
|
|
"""Helper: create a project in CREATED state and return the project path."""
|
|
project_dir = str(create_workspace(name))
|
|
manifest = create_manifest(name)
|
|
save_manifest(project_dir, manifest)
|
|
return project_dir
|
|
|
|
|
|
def _make_imported_project(name: str, binary_path: str = "/fake/test.exe") -> str:
|
|
"""Helper: create a project in IMPORTED state with a binary record."""
|
|
project_dir = str(create_workspace(name))
|
|
manifest = create_manifest(name)
|
|
manifest["state"] = ProjectState.IMPORTED.value
|
|
manifest["binary_count"] = 1
|
|
# Store binary record
|
|
binary_id = str(UUID(int=1))
|
|
binary_record = {
|
|
"id": binary_id,
|
|
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
|
|
"path": binary_path,
|
|
"format": "PE",
|
|
"import_mode": "copy",
|
|
"size_bytes": 512,
|
|
"architecture": "x86",
|
|
}
|
|
manifest["current_binary"] = binary_record
|
|
# Write binary record file
|
|
binaries_dir = os.path.join(project_dir, "binaries")
|
|
os.makedirs(binaries_dir, exist_ok=True)
|
|
with open(os.path.join(binaries_dir, f"{binary_id}.json"), "w") as f:
|
|
json.dump(binary_record, f)
|
|
save_manifest(project_dir, manifest)
|
|
return project_dir
|
|
|
|
|
|
def _make_analyzing_project(name: str) -> str:
|
|
"""Helper: create a project in ANALYZING state with a lock."""
|
|
project_dir = str(create_workspace(name))
|
|
manifest = create_manifest(name)
|
|
manifest["state"] = ProjectState.ANALYZING.value
|
|
manifest["binary_count"] = 1
|
|
binary_id = str(UUID(int=2))
|
|
manifest["current_binary"] = {
|
|
"id": binary_id,
|
|
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
|
|
"path": "/fake/test.exe",
|
|
"format": "PE",
|
|
"import_mode": "copy",
|
|
"size_bytes": 512,
|
|
"architecture": "x86",
|
|
}
|
|
save_manifest(project_dir, manifest)
|
|
# Create lock file
|
|
lock_path = os.path.join(project_dir, "project.lock")
|
|
with open(lock_path, "w") as f:
|
|
f.write(f"pid={os.getpid()} host=test purpose=analysis acquired_at=now")
|
|
return project_dir
|
|
|
|
|
|
def _make_ready_project(name: str, binary_path: str = "/fake/test.exe") -> str:
|
|
"""Helper: create a project in READY state."""
|
|
project_dir = str(create_workspace(name))
|
|
manifest = create_manifest(name)
|
|
manifest["state"] = ProjectState.READY.value
|
|
manifest["binary_count"] = 1
|
|
manifest["is_stale"] = False
|
|
binary_id = str(UUID(int=3))
|
|
# Create sample file first to compute its SHA-256
|
|
samples_dir = os.path.join(project_dir, "samples")
|
|
os.makedirs(samples_dir, exist_ok=True)
|
|
sample_content = b"MZ\x00\x01" + b"\x00" * 508 # 512 bytes
|
|
with open(os.path.join(samples_dir, binary_id), "wb") as f:
|
|
f.write(sample_content)
|
|
actual_sha = hashlib.sha256(sample_content).hexdigest()
|
|
manifest["current_binary"] = {
|
|
"id": binary_id,
|
|
"sha256": actual_sha,
|
|
"path": binary_path,
|
|
"format": "PE",
|
|
"import_mode": "copy",
|
|
"size_bytes": 512,
|
|
"architecture": "x86",
|
|
}
|
|
manifest["analysis_profile"] = "standard"
|
|
save_manifest(project_dir, manifest)
|
|
binaries_dir = os.path.join(project_dir, "binaries")
|
|
os.makedirs(binaries_dir, exist_ok=True)
|
|
with open(os.path.join(binaries_dir, f"{binary_id}.json"), "w") as f:
|
|
json.dump(manifest["current_binary"], f)
|
|
return project_dir
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Import tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestImportCopyMode:
|
|
"""VAL-IMP-001: Import copy mode produces JSON envelope with binary identity."""
|
|
|
|
def test_import_copy_mode_returns_identity(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""Import in copy mode returns binary_id, sha256, path, format, import_mode, size_bytes."""
|
|
_make_created_project("imp-test")
|
|
exit_code, result = _capture_json(["import", test_binary, "--project", "imp-test"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
data = result["data"]
|
|
assert "binary_id" in data
|
|
assert "binary_sha256" in data
|
|
assert "binary_path" in data
|
|
assert "format" in data
|
|
assert data["import_mode"] == "copy"
|
|
assert "size_bytes" in data
|
|
|
|
# Verify UUID format for binary_id
|
|
UUID(data["binary_id"])
|
|
|
|
# Verify SHA-256 is 64 hex chars
|
|
assert len(data["binary_sha256"]) == 64
|
|
assert all(c in "0123456789abcdef" for c in data["binary_sha256"])
|
|
|
|
def test_import_copy_mode_copies_file(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-006: Copy mode copies file to samples/<binary-id>."""
|
|
_make_created_project("copy-test")
|
|
exit_code, result = _capture_json(["import", test_binary, "--project", "copy-test"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
binary_id = result["data"]["binary_id"]
|
|
|
|
# Check that sample file exists
|
|
project_dir = get_project_path("copy-test")
|
|
sample_path = os.path.join(str(project_dir), "samples", binary_id)
|
|
assert os.path.exists(sample_path)
|
|
|
|
# Verify SHA-256 matches
|
|
with open(sample_path, "rb") as f:
|
|
content = f.read()
|
|
sha256 = hashlib.sha256(content).hexdigest()
|
|
assert sha256 == result["data"]["binary_sha256"]
|
|
|
|
def test_import_updates_project_state(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""Import transitions project from CREATED to IMPORTED."""
|
|
_make_created_project("state-test")
|
|
exit_code, _ = _capture_json(["import", test_binary, "--project", "state-test"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
project_dir = str(get_project_path("state-test"))
|
|
manifest = load_manifest(project_dir)
|
|
assert manifest["state"] == ProjectState.IMPORTED.value
|
|
assert manifest["binary_count"] == 1
|
|
|
|
def test_import_sets_sha256_before_backend(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-003: SHA-256 computed client-side, present even on import failure."""
|
|
_make_created_project("sha-before-backend")
|
|
|
|
# The SHA-256 should match the pre-computed hash
|
|
precomputed = hashlib.sha256(Path(test_binary).read_bytes()).hexdigest()
|
|
|
|
exit_code, result = _capture_json(
|
|
["import", test_binary, "--project", "sha-before-backend"], capsys
|
|
)
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["data"]["binary_sha256"] == precomputed
|
|
|
|
|
|
class TestImportReferenceMode:
|
|
"""VAL-IMP-002: Import reference mode tracks source path and detects staleness."""
|
|
|
|
def test_import_reference_mode(self, test_binary: str, capsys: pytest.CaptureFixture) -> None:
|
|
"""Import in reference mode sets import_mode=reference, tracks external path."""
|
|
_make_created_project("ref-import")
|
|
exit_code, result = _capture_json(
|
|
["import", test_binary, "--project", "ref-import", "--reference"], capsys
|
|
)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
data = result["data"]
|
|
assert data["import_mode"] == "reference"
|
|
assert data["binary_path"] == test_binary
|
|
|
|
def test_reference_mode_no_copy(self, test_binary: str, capsys: pytest.CaptureFixture) -> None:
|
|
"""VAL-IMP-006: Reference mode does not copy file to samples/."""
|
|
_make_created_project("ref-no-copy")
|
|
exit_code, result = _capture_json(
|
|
["import", test_binary, "--project", "ref-no-copy", "--reference"], capsys
|
|
)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
project_dir = str(get_project_path("ref-no-copy"))
|
|
samples_dir = os.path.join(project_dir, "samples")
|
|
# samples dir may exist but should be empty of the binary-id file
|
|
binary_id = result["data"]["binary_id"]
|
|
sample_path = os.path.join(samples_dir, binary_id)
|
|
assert not os.path.exists(sample_path)
|
|
|
|
def test_reference_mode_staleness_on_source_change(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-002: Staleness detected after source change in reference mode."""
|
|
project_dir = _make_ready_project("staleness-ref", binary_path=test_binary)
|
|
# Update the manifest to simulate reference mode import
|
|
manifest = load_manifest(project_dir)
|
|
manifest["current_binary"]["import_mode"] = "reference"
|
|
manifest["is_stale"] = False
|
|
manifest["state"] = ProjectState.READY.value
|
|
save_manifest(project_dir, manifest)
|
|
|
|
# Now modify the source file
|
|
Path(test_binary).write_bytes(Path(test_binary).read_bytes() + b"\x00")
|
|
|
|
# Analyze should detect staleness
|
|
_exit_code, result = _capture_json(["analyze", "--project", "staleness-ref"], capsys)
|
|
|
|
# Should report staleness (not proceed to analyze automatically)
|
|
assert result["provenance"].get("project_state") == "STALE"
|
|
assert any(
|
|
"stale" in str(d.get("message", "")).lower()
|
|
or "sha" in str(d.get("message", "")).lower()
|
|
for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
|
|
class TestImportErrors:
|
|
"""VAL-IMP-004, VAL-IMP-005, VAL-IMP-007, VAL-IMP-016, VAL-IMP-019."""
|
|
|
|
def test_import_unsupported_format(
|
|
self, unsupported_file: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-004: Unsupported format rejected with exit code 5."""
|
|
_make_created_project("bad-fmt")
|
|
exit_code, result = _capture_json(
|
|
["import", unsupported_file, "--project", "bad-fmt"], capsys
|
|
)
|
|
|
|
assert exit_code == ExitCode.UNSUPPORTED_FORMAT
|
|
assert result["success"] is False
|
|
assert any(
|
|
"format" in str(d.get("message", "")).lower()
|
|
or "unsupported" in str(d.get("message", "")).lower()
|
|
for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
def test_import_nonexistent_project(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-007: Import into non-existent project exits with code 6."""
|
|
exit_code, result = _capture_json(
|
|
["import", test_binary, "--project", "nonexistent-proj"], capsys
|
|
)
|
|
|
|
assert exit_code == ExitCode.PROJECT_NOT_FOUND
|
|
assert result["success"] is False
|
|
|
|
def test_import_rejects_binary_above_max_size(
|
|
self, large_binary: str, small_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-005: Binary above max size rejected with non-zero exit."""
|
|
# Create project with max_binary_size_bytes = 64 (small)
|
|
project_dir = _make_created_project("max-size")
|
|
manifest = load_manifest(project_dir)
|
|
manifest["max_binary_size_bytes"] = 64
|
|
save_manifest(project_dir, manifest)
|
|
|
|
# Try to import the large binary (512 bytes)
|
|
exit_code, result = _capture_json(["import", large_binary, "--project", "max-size"], capsys)
|
|
|
|
assert exit_code != ExitCode.SUCCESS
|
|
assert result["success"] is False
|
|
assert any(
|
|
"size" in str(d.get("message", "")).lower()
|
|
or "limit" in str(d.get("message", "")).lower()
|
|
for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
def test_import_during_active_analysis(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-019: Import during active analysis is rejected."""
|
|
_make_analyzing_project("busy-proj")
|
|
exit_code, result = _capture_json(["import", test_binary, "--project", "busy-proj"], capsys)
|
|
|
|
assert exit_code != ExitCode.SUCCESS
|
|
assert result["success"] is False
|
|
assert any(
|
|
"lock" in str(d.get("message", "")).lower()
|
|
or "busy" in str(d.get("message", "")).lower()
|
|
or "analyzing" in str(d.get("message", "")).lower()
|
|
for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
def test_import_backend_failure(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""VAL-IMP-016: Import backend failure exits with code 10."""
|
|
_make_created_project("imp-fail")
|
|
_exit_code, _result = _capture_json(
|
|
["import", test_binary, "--project", "imp-fail"], capsys
|
|
)
|
|
|
|
# The real import should succeed. We test this differently -
|
|
# by checking that when backend raises ImportFailedError, exit code is 10.
|
|
# For the fake adapter, we'd need to configure import failure.
|
|
# Since the dispatcher doesn't directly expose adapter config, we test
|
|
# the error code routing via the existing error hierarchy.
|
|
from binary_analysis.domain.errors import ImportFailedError
|
|
|
|
e = ImportFailedError("Backend connection lost", binary_path=test_binary)
|
|
assert e.exit_code == ExitCode.IMPORT_FAILED
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Analyze tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAnalyzeStateTransitions:
|
|
"""VAL-IMP-008: Analyze transitions project state through lock lifecycle."""
|
|
|
|
def test_analyze_transitions_imported_to_ready(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""Analyze transitions IMPORTED -> ANALYZING -> READY."""
|
|
_make_imported_project("analyze-transition", test_binary)
|
|
exit_code, result = _capture_json(["analyze", "--project", "analyze-transition"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["provenance"].get("project_state") == "READY"
|
|
|
|
# Verify manifest reflects READY state
|
|
project_dir = str(get_project_path("analyze-transition"))
|
|
manifest = load_manifest(project_dir)
|
|
assert manifest["state"] == ProjectState.READY.value
|
|
|
|
def test_analyze_lock_released_after_completion(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""Lock is released after successful analysis."""
|
|
_make_imported_project("lock-release", test_binary)
|
|
_capture_json(["analyze", "--project", "lock-release"], capsys)
|
|
|
|
# Verify lock is released
|
|
project_dir = str(get_project_path("lock-release"))
|
|
assert not is_locked(project_dir)
|
|
|
|
|
|
class TestAnalyzeProfiles:
|
|
"""VAL-IMP-011: Analyze with unknown profile reports available profiles."""
|
|
|
|
def test_analyze_standard_profile(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""Analyze with standard profile succeeds."""
|
|
_make_imported_project("std-profile", test_binary)
|
|
exit_code, result = _capture_json(
|
|
["analyze", "--project", "std-profile", "--profile", "standard"], capsys
|
|
)
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["provenance"].get("analysis_profile") == "standard"
|
|
|
|
def test_analyze_quick_profile(self, test_binary: str, capsys: pytest.CaptureFixture) -> None:
|
|
"""Analyze with quick profile succeeds."""
|
|
_make_imported_project("quick-profile", test_binary)
|
|
exit_code, result = _capture_json(
|
|
["analyze", "--project", "quick-profile", "--profile", "quick"], capsys
|
|
)
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["provenance"].get("analysis_profile") == "quick"
|
|
|
|
def test_analyze_deep_profile(self, test_binary: str, capsys: pytest.CaptureFixture) -> None:
|
|
"""Analyze with deep profile succeeds."""
|
|
_make_imported_project("deep-profile", test_binary)
|
|
exit_code, result = _capture_json(
|
|
["analyze", "--project", "deep-profile", "--profile", "deep"], capsys
|
|
)
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["provenance"].get("analysis_profile") == "deep"
|
|
|
|
def test_analyze_unknown_profile(self, test_binary: str, capsys: pytest.CaptureFixture) -> None:
|
|
"""VAL-IMP-011: Unknown profile rejected with list of available profiles."""
|
|
_make_imported_project("bad-profile", test_binary)
|
|
exit_code, result = _capture_json(
|
|
["analyze", "--project", "bad-profile", "--profile", "nonexistent"], capsys
|
|
)
|
|
|
|
assert exit_code != ExitCode.SUCCESS
|
|
assert result["success"] is False
|
|
# Should mention available profiles
|
|
diagnostics_str = json.dumps(result.get("diagnostics", []))
|
|
assert any(p in diagnostics_str for p in ["standard", "quick", "deep"])
|
|
|
|
|
|
class TestAnalyzeErrors:
|
|
"""VAL-IMP-009, VAL-IMP-014, VAL-IMP-017, VAL-IMP-018."""
|
|
|
|
def test_analyze_on_created_only_project(self, capsys: pytest.CaptureFixture) -> None:
|
|
"""VAL-IMP-014: Analyze on CREATED-only project exits with code 7."""
|
|
_make_created_project("no-binary")
|
|
exit_code, result = _capture_json(["analyze", "--project", "no-binary"], capsys)
|
|
|
|
assert exit_code == ExitCode.BINARY_NOT_FOUND
|
|
assert result["success"] is False
|
|
assert any(
|
|
"binary" in str(d.get("message", "")).lower()
|
|
or "import" in str(d.get("message", "")).lower()
|
|
for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
def test_analyze_hard_failure(self, test_binary: str, capsys: pytest.CaptureFixture) -> None:
|
|
"""VAL-IMP-017: Hard analysis failure exits with code 11, state=FAILED."""
|
|
from binary_analysis.domain.errors import AnalysisFailedError
|
|
|
|
_make_imported_project("hard-fail", test_binary)
|
|
_exit_code, _result = _capture_json(["analyze", "--project", "hard-fail"], capsys)
|
|
|
|
# Real analyze succeeds with fake adapter, so test the error directly
|
|
e = AnalysisFailedError("Complete analysis crash", project="hard-fail")
|
|
assert e.exit_code == ExitCode.ANALYSIS_FAILED
|
|
|
|
def test_backend_failure_exit_code_13(self) -> None:
|
|
"""VAL-IMP-018: Backend crash during query exits with code 13."""
|
|
from binary_analysis.domain.errors import BackendFailureError
|
|
|
|
e = BackendFailureError("Backend crashed", original_error="Segmentation fault")
|
|
assert e.exit_code == ExitCode.BACKEND_FAILURE
|
|
|
|
|
|
class TestAnalyzeStaleness:
|
|
"""VAL-IMP-010, VAL-IMP-015: Staleness detection."""
|
|
|
|
def test_analyze_staleness_after_source_change(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-010: Staleness detected after source change; does not re-analyze automatically."""
|
|
project_dir = _make_ready_project("stale-source", test_binary)
|
|
# Modify the sample file to simulate source change
|
|
binary_id = load_manifest(project_dir)["current_binary"]["id"]
|
|
sample_path = os.path.join(project_dir, "samples", binary_id)
|
|
if os.path.exists(sample_path):
|
|
with open(sample_path, "ab") as f:
|
|
f.write(b"\x00modified")
|
|
|
|
_exit_code, result = _capture_json(["analyze", "--project", "stale-source"], capsys)
|
|
|
|
# Should detect staleness, not proceed to full re-analysis
|
|
assert result["provenance"].get("project_state") == "STALE"
|
|
assert any(
|
|
"stale" in str(d.get("message", "")).lower() for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
def test_analyze_profile_change_detects_staleness(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-015: Profile change detected as staleness trigger."""
|
|
project_dir = _make_ready_project("profile-stale", test_binary)
|
|
manifest = load_manifest(project_dir)
|
|
manifest["analysis_profile"] = "quick" # Was analyzed with quick
|
|
save_manifest(project_dir, manifest)
|
|
|
|
_exit_code, result = _capture_json(
|
|
["analyze", "--project", "profile-stale", "--profile", "standard"], capsys
|
|
)
|
|
|
|
# Should detect profile change as staleness
|
|
assert result["provenance"].get("project_state") == "STALE"
|
|
assert any(
|
|
"profile" in str(d.get("message", "")).lower()
|
|
or "stale" in str(d.get("message", "")).lower()
|
|
for d in result.get("diagnostics", [])
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Metadata tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestMetadata:
|
|
"""VAL-IMP-012, VAL-IMP-013: Metadata command."""
|
|
|
|
def test_metadata_returns_canonical_fields(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-012: Metadata returns format, architecture, endianness, size_bytes, entry_point."""
|
|
_make_imported_project("meta-canonical", test_binary)
|
|
exit_code, result = _capture_json(["metadata", "--project", "meta-canonical"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
data = result["data"]
|
|
assert "format" in data
|
|
assert "architecture" in data
|
|
assert "endianness" in data
|
|
assert "size_bytes" in data
|
|
assert "entry_point" in data or data.get("entry_point") is not None
|
|
|
|
def test_metadata_no_backend_specific_keys(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-012: No backend-specific keys at root of data."""
|
|
_make_imported_project("meta-no-backend", test_binary)
|
|
exit_code, result = _capture_json(["metadata", "--project", "meta-no-backend"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
data = result["data"]
|
|
# Only canonical fields should be at root
|
|
allowed_keys = {
|
|
"format",
|
|
"architecture",
|
|
"endianness",
|
|
"size_bytes",
|
|
"entry_point",
|
|
"compiler",
|
|
"source_language",
|
|
}
|
|
for key in data:
|
|
assert key in allowed_keys, f"Non-canonical key in metadata: {key}"
|
|
|
|
def test_metadata_reports_project_state(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-013: Metadata reports project_state in provenance regardless of analysis state."""
|
|
_make_imported_project("meta-state", test_binary)
|
|
exit_code, result = _capture_json(["metadata", "--project", "meta-state"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert "project_state" in result.get("provenance", {})
|
|
assert result["provenance"]["project_state"] in ("IMPORTED", "READY")
|
|
|
|
def test_metadata_on_unanalyzed_project(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""VAL-IMP-013: Metadata returns data even when project hasn't been analyzed."""
|
|
_make_imported_project("meta-unanalyzed", test_binary)
|
|
exit_code, result = _capture_json(["metadata", "--project", "meta-unanalyzed"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["success"] is True
|
|
data = result["data"]
|
|
assert data.get("format") is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Edge cases
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestImportEdgeCases:
|
|
"""Additional edge cases for import."""
|
|
|
|
def test_import_missing_project_flag(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""Import without --project should fail - argparse rejects it."""
|
|
# argparse exits with code 2 when required arg is missing
|
|
# main() translates SystemExit to return code 2
|
|
exit_code = main(["--json", "import", test_binary])
|
|
assert exit_code == ExitCode.INVALID_ARGS
|
|
|
|
def test_import_missing_binary_path(self, capsys: pytest.CaptureFixture) -> None:
|
|
"""Import without binary path should fail."""
|
|
exit_code, _result = _capture_json(["import", "--project", "test"], capsys)
|
|
assert exit_code == ExitCode.INVALID_ARGS
|
|
|
|
def test_import_nonexistent_file(self, capsys: pytest.CaptureFixture) -> None:
|
|
"""Import of non-existent file path should fail."""
|
|
_make_created_project("bad-file")
|
|
exit_code, result = _capture_json(
|
|
["import", "/nonexistent/path/to/binary.exe", "--project", "bad-file"], capsys
|
|
)
|
|
assert exit_code != ExitCode.SUCCESS
|
|
assert result["success"] is False
|
|
|
|
|
|
class TestAnalyzeEdgeCases:
|
|
"""Additional edge cases for analyze."""
|
|
|
|
def test_analyze_missing_project_flag(self, capsys: pytest.CaptureFixture) -> None:
|
|
"""Analyze without --project should fail."""
|
|
exit_code, _result = _capture_json(["analyze"], capsys)
|
|
assert exit_code != ExitCode.SUCCESS
|
|
|
|
def test_analyze_stale_to_analyzing_transition(
|
|
self, test_binary: str, capsys: pytest.CaptureFixture
|
|
) -> None:
|
|
"""STALE state allows analysis (re-analysis)."""
|
|
project_dir = _make_ready_project("stale-reanalyze", test_binary)
|
|
# Set to STALE
|
|
manifest = load_manifest(project_dir)
|
|
manifest["state"] = ProjectState.STALE.value
|
|
save_manifest(project_dir, manifest)
|
|
|
|
exit_code, result = _capture_json(["analyze", "--project", "stale-reanalyze"], capsys)
|
|
|
|
assert exit_code == ExitCode.SUCCESS
|
|
assert result["provenance"].get("project_state") == "READY"
|
|
|
|
|
|
class TestMetadataEdgeCases:
|
|
"""Additional edge cases for metadata."""
|
|
|
|
def test_metadata_nonexistent_project(self, capsys: pytest.CaptureFixture) -> None:
|
|
"""Metadata on non-existent project fails."""
|
|
exit_code, _result = _capture_json(["metadata", "--project", "nonexistent"], capsys)
|
|
assert exit_code == ExitCode.PROJECT_NOT_FOUND
|
|
|
|
def test_metadata_on_created_project(self, capsys: pytest.CaptureFixture) -> None:
|
|
"""Metadata on CREATED project (no binary) fails with exit code 7."""
|
|
_make_created_project("no-bin-meta")
|
|
exit_code, _result = _capture_json(["metadata", "--project", "no-bin-meta"], capsys)
|
|
assert exit_code == ExitCode.BINARY_NOT_FOUND
|