From caa46d9c057d61207fc46952fd2faea02329c2f2 Mon Sep 17 00:00:00 2001 From: Magnus Hedemark Date: Thu, 18 Jun 2026 12:18:35 -0400 Subject: [PATCH] feat: add open-knowledge-format skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Google's Open Knowledge Format (OKF) v0.1 — an open, vendor-neutral spec for representing knowledge as markdown files with YAML frontmatter, designed for AI agent consumption. - SKILL.md: core instructions, frontmatter spec, conformance criteria, cross-linking rules, relationship to other formats, quick start - references/spec-summary.md: full OKF v0.1 specification reference - references/bundle-architecture.md: directory structure and conventions - references/use-cases.md: real-world adoption and patterns - scripts/okf-bundle-validate.py: validate OKF bundle structure - assets/concept-template.md: template for creating concept files - assets/example-bundle/: minimal conformant OKF bundle --- AGENTS.md | 1 + README.md | 4 + open-knowledge-format/SKILL.md | 169 +++++++++ .../assets/concept-template.md | 38 ++ .../assets/example-bundle/datasets/index.md | 3 + .../assets/example-bundle/datasets/sales.md | 9 + .../assets/example-bundle/index.md | 6 + .../assets/example-bundle/tables/customers.md | 20 ++ .../assets/example-bundle/tables/index.md | 4 + .../assets/example-bundle/tables/orders.md | 18 + .../references/bundle-architecture.md | 145 ++++++++ .../references/spec-summary.md | 165 +++++++++ open-knowledge-format/references/use-cases.md | 78 ++++ .../scripts/okf-bundle-validate.py | 336 ++++++++++++++++++ 14 files changed, 996 insertions(+) create mode 100644 open-knowledge-format/SKILL.md create mode 100644 open-knowledge-format/assets/concept-template.md create mode 100644 open-knowledge-format/assets/example-bundle/datasets/index.md create mode 100644 open-knowledge-format/assets/example-bundle/datasets/sales.md create mode 100644 open-knowledge-format/assets/example-bundle/index.md create mode 100644 open-knowledge-format/assets/example-bundle/tables/customers.md create mode 100644 open-knowledge-format/assets/example-bundle/tables/index.md create mode 100644 open-knowledge-format/assets/example-bundle/tables/orders.md create mode 100644 open-knowledge-format/references/bundle-architecture.md create mode 100644 open-knowledge-format/references/spec-summary.md create mode 100644 open-knowledge-format/references/use-cases.md create mode 100644 open-knowledge-format/scripts/okf-bundle-validate.py diff --git a/AGENTS.md b/AGENTS.md index 2d8c5d0..d27ef84 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,6 +66,7 @@ When the user mentions these keywords, load the corresponding skill: | "skill format", "how do I make a skill", "agentskills.io" | [agent-skills](agent-skills/SKILL.md) | | "last.fm", "scrobble", "music discovery", "listening history", "similar artists", "lastfm", "weekly top artists", "genre charts" | [lastfm](lastfm/SKILL.md) | | "nous", "theia", "hermes brand", "brand identity", "style guide", "mascot", "anime style", "cyber-classical", "color palette reference" | [nous-branding](nous-branding/SKILL.md) | +| "okf", "open knowledge format", "knowledge bundle", "LLM wiki", "agent knowledge", "Google knowledge format", "markdown knowledge", "vendor-neutral knowledge", "create an OKF bundle", "validate OKF", "concept document", "knowledge format" | [open-knowledge-format](open-knowledge-format/SKILL.md) | | "raleigh", "open data", "city of raleigh", "raleigh data", "crime data", "food inspections", "building permits", "arcgis", "public data", "raleighnc" | [raleigh](raleigh/SKILL.md) | | "open source", "contributing", "how to contribute", "submit a PR", "file an issue", "CONTRIBUTING.md", "bug report template", "PR template" | [opensource-contributions](opensource-contributions/SKILL.md) | | "workflow", "figure out my workflow", "analyze my process", "what do I actually do", "catalog my workflow", "formalize my process", "workflow architect", "onboard me to my own process" | [workflow-architect](bundles/workflow-architect/SKILL.md) | diff --git a/README.md b/README.md index 7d38f10..d43c745 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,10 @@ prompt templates for text-only and reference-image-driven workflows. > **Cross-pollination note:** This skill was developed in parallel with [plntrprotocol/nous-branding](https://github.com/plntrprotocol/nous-branding) — a sibling project created independently, in friendship, for the same purpose. We've been learning from each other's approaches throughout development, and each repo has strengths the other doesn't. If you're using ours, go check out theirs too. +### [open-knowledge-format](open-knowledge-format/SKILL.md) + +Google's Open Knowledge Format (OKF) v0.1 — create, validate, and consume vendor-neutral AI agent knowledge bundles. Markdown files with YAML frontmatter, organized in directory hierarchies with cross-links and progressive disclosure. Ships a validation script, concept template, example bundle, and detailed references covering the spec, bundle architecture, and real-world use cases. + ### [opensource-contributions](opensource-contributions/SKILL.md) Comprehensive open source contribution guidance — from reading CONTRIBUTING.md and filing good bug reports through branching, committing, PR creation, and the release cycle. Covers both contributor and maintainer workflows with progressive disclosure: a concise orchestrator SKILL.md loads detailed phase references on demand. Includes a portable PR template compliance checker script. Agent disclosure template for AI-assisted contributions. diff --git a/open-knowledge-format/SKILL.md b/open-knowledge-format/SKILL.md new file mode 100644 index 0000000..03ee6d7 --- /dev/null +++ b/open-knowledge-format/SKILL.md @@ -0,0 +1,169 @@ +--- +name: open-knowledge-format +description: >- + Google's Open Knowledge Format (OKF) v0.1 — an open, vendor-neutral spec for + representing knowledge as markdown files with YAML frontmatter, designed for + AI agent consumption. Use when the user mentions OKF, Open Knowledge Format, + Google's knowledge format, LLM wiki bundles, agent knowledge packs, creating + OKF bundles, validating OKF documents, or converting knowledge into the OKF + standard. +license: MIT +compatibility: >- + Portable across any AgentSkills-compatible harness. The validation script + requires Python 3.8+ with PyYAML (pip install PyYAML). All examples use + standard markdown — readable in any editor or terminal. +metadata: + spec-version: "1.0" + skills: [okf, open-knowledge-format, knowledge-format, agent-knowledge, llm-wiki] + tags: [okf, knowledge-format, google, ai-agents, markdown, knowledge-management] +--- + +# Open Knowledge Format (OKF) — v0.1 + +Google Cloud published the [Open Knowledge Format v0.1](https://cloud.google.com/blog/products/data-analytics/how-the-open-knowledge-format-can-improve-data-sharing/) on **June 12, 2026** — an open specification that formalizes the [LLM-wiki pattern](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) into a portable, vendor-neutral format for AI agent knowledge. + +**The design is intentionally minimal:** a directory of markdown files with YAML frontmatter. No schema registry, no central authority, no required tooling. + +> "If you can `cat` a file, you can read OKF; if you can `git clone` a repo, you can ship it." + +## Core Concepts + +| Concept | Definition | +|---------|------------| +| **Knowledge Bundle** | A self-contained directory tree of markdown files. The unit of distribution. | +| **Concept** | A single unit of knowledge — one markdown file. May describe a table, a metric, a playbook, an API, or any idea. | +| **Concept ID** | The file path with `.md` stripped (e.g. `tables/users.md` has ID `tables/users`). | +| **Frontmatter** | YAML block at the top of each file with structured metadata. | +| **Body** | Standard markdown content after the frontmatter. | +| **Link** | A markdown link from one concept to another — expresses relationships. | + +## Reference Files + +| Reference | Load when | File | +|-----------|-----------|------| +| Spec summary | You need the full OKF v0.1 specification, conformance criteria, and field definitions | `references/spec-summary.md` | +| Bundle architecture | You're creating or structuring an OKF bundle — directory layout, index files, cross-linking conventions | `references/bundle-architecture.md` | +| Use cases | You need real-world examples, third-party implementations, and adoption patterns | `references/use-cases.md` | + +## Scripts + +| Script | Purpose | Load when | +|--------|---------|-----------| +| `okf-bundle-validate.py` | Validate an OKF bundle directory — checks frontmatter, required fields, reserved filenames, and cross-link integrity | You've created or modified an OKF bundle and want to verify conformance | + +## Assets + +| Asset | Purpose | Path | +|-------|---------|------| +| Concept template | A template markdown file with all recommended frontmatter fields and body sections | `assets/concept-template.md` | +| Example bundle | A minimal but complete OKF bundle showing directory structure, index files, and cross-linked concepts | `assets/example-bundle/` | + +## Bundle Structure + +``` +path/to/bundle/ +├── index.md # Optional. Directory listing for progressive disclosure. +├── log.md # Optional. Chronological update history. +├── .md # A concept at the bundle root. +└── / # Subdirectories organize concepts into groups. + ├── index.md + ├── .md + └── / + └── … +``` + +Files named `index.md` and `log.md` are **reserved** — they have defined semantics and must not be used for concept documents. + +## Frontmatter Fields + +```yaml +--- +type: # REQUIRED +title: +description: +resource: +tags: [, , …] # Optional +timestamp: # Optional last-modified time +# … other producer-defined key/value pairs +--- +``` + +**Required:** `type` — a short string identifying the kind of concept. Types are NOT registered centrally. Producers pick descriptive values; consumers tolerate unknown types gracefully. + +**Conventional body sections:** + +| Heading | Purpose | +|---------|---------| +| `# Schema` | Structured description of an asset's columns/fields | +| `# Examples` | Concrete usage examples, often as code blocks | +| `# Citations` | External sources backing claims in the body | + +## Cross-linking + +Concepts link to each other via standard markdown links: + +- **Absolute (bundle-relative):** Begin with `/`, relative to bundle root. Recommended — stable when documents move. +- **Relative:** Standard markdown relative paths. + +A link from concept A to concept B asserts a relationship. The specific kind (references, depends-on, joins-with) is conveyed by the surrounding prose, not by the link syntax. + +**Consumers MUST tolerate broken links** — a link whose target doesn't exist is not malformed; it may represent not-yet-written knowledge. + +## Conformance + +A bundle is **conformant** with OKF v0.1 if: + +1. Every non-reserved `.md` file contains parseable YAML frontmatter +2. Every frontmatter block contains a non-empty `type` field +3. Reserved filenames (`index.md`, `log.md`) follow their defined structure + +**Consumers MUST NOT reject a bundle for:** missing optional frontmatter fields, unknown `type` values, unknown frontmatter keys, broken cross-links, or missing `index.md` files. + +## Relationship to Other Formats + +| Format | How OKF Differs | +|--------|----------------| +| **Karpathy's LLM Wiki** | OKF specifies the interoperability surface — required fields, reserved filenames, conformance criteria | +| **Obsidian / Notion vaults** | OKF is format-only — no tooling, no runtime, no UI. Any editor works | +| **AGENTS.md / CLAUDE.md** | OKF is multi-file, hierarchical, cross-linked — not a single convention file | +| **Metadata-as-code repos** | OKF standardizes the file format so different producers and consumers interoperate | +| **Domain schemas (Avro, Protobuf, OpenAPI)** | OKF references them — it does not subsume or replace them | + +## Quick Start + +```bash +# 1. Create a bundle directory +mkdir my-knowledge && cd my-knowledge + +# 2. Create a concept file +cat > datasets/sales.md << 'EOF' +--- +type: BigQuery Dataset +title: Sales +description: All sales-related tables for the retail business. +tags: [sales] +timestamp: 2026-06-18T00:00:00Z +--- +The sales dataset contains [orders](/tables/orders.md) and [customers](/tables/customers.md). +EOF + +# 3. Add an index for navigation +cat > index.md << 'EOF' +# Knowledge Bundle + +* [Sales Dataset](datasets/sales.md) - Retail sales data +EOF + +# 4. Validate your bundle +python3 scripts/okf-bundle-validate.py . +``` + +## Gotchas + +- **Type values are not registered centrally.** Pick descriptive types (`BigQuery Table`, `Playbook`, `Metric`). Consumers should handle unknown types gracefully. +- **File path is the identity.** Renaming a file changes its concept ID. Use stable paths or add redirects in documentation. +- **`index.md` uses no frontmatter** (except optionally at the bundle root for `okf_version`). The body is a markdown list with links. +- **`log.md` date headings must be ISO 8601** (`YYYY-MM-DD`). +- **Cross-links are untyped.** The relationship semantics live in the prose, not in the link syntax. Graph builders treat all links as directed edges. +- **OKF does not specify a tag-browsing format.** Producers that want tag aggregation should synthesize it at consumption time by scanning frontmatter. +- **OKF v0.1 is a draft.** The spec will evolve. Minor bumps add backward-compatible features; major bumps may break required fields. Consumers should do best-effort consumption on unknown versions. diff --git a/open-knowledge-format/assets/concept-template.md b/open-knowledge-format/assets/concept-template.md new file mode 100644 index 0000000..7c839ad --- /dev/null +++ b/open-knowledge-format/assets/concept-template.md @@ -0,0 +1,38 @@ +--- +type: # REQUIRED — e.g. "BigQuery Table", "Metric", "Playbook", "API Endpoint" +title: # Recommended +description: # Recommended — used in index.md entries and search snippets +resource: # Optional — URL for the underlying asset +tags: [, ] # Optional — cross-cutting categorization +timestamp: # Optional — ISO 8601 last-modified time +--- + +# Concept Title + +A short paragraph explaining what this concept is and why it matters. What problem does it solve? When would someone reach for this? + +## Schema + +If this concept describes a structured asset (table, API, dataset), document its fields here. + +| Field | Type | Description | +|-------|------|-------------| +| `field_name` | TYPE | Description of the field | +| `foreign_key` | TYPE | FK to [related concept](/path/to/concept.md) | + +## Examples + +Usage examples as code blocks or scenarios. + +```sql +SELECT field_name FROM table WHERE condition; +``` + +## References + +* [Related concept 1](/path/to/concept-1.md) +* [Related concept 2](/path/to/concept-2.md) + +## Citations + +[1] [External source](https://example.com) diff --git a/open-knowledge-format/assets/example-bundle/datasets/index.md b/open-knowledge-format/assets/example-bundle/datasets/index.md new file mode 100644 index 0000000..d54096b --- /dev/null +++ b/open-knowledge-format/assets/example-bundle/datasets/index.md @@ -0,0 +1,3 @@ +# Datasets + +* [Sales](sales.md) - Retail sales data diff --git a/open-knowledge-format/assets/example-bundle/datasets/sales.md b/open-knowledge-format/assets/example-bundle/datasets/sales.md new file mode 100644 index 0000000..4376211 --- /dev/null +++ b/open-knowledge-format/assets/example-bundle/datasets/sales.md @@ -0,0 +1,9 @@ +--- +type: BigQuery Dataset +title: Sales +description: All sales-related tables for the retail business. +tags: [sales] +timestamp: 2026-06-18T00:00:00Z +--- + +The sales dataset contains transactional tables, including [orders](/tables/orders.md) and [customers](/tables/customers.md). diff --git a/open-knowledge-format/assets/example-bundle/index.md b/open-knowledge-format/assets/example-bundle/index.md new file mode 100644 index 0000000..1bc7a2f --- /dev/null +++ b/open-knowledge-format/assets/example-bundle/index.md @@ -0,0 +1,6 @@ +# Example OKF Bundle + +A minimal knowledge bundle demonstrating the OKF v0.1 structure. + +* [Datasets](/datasets/index.md) - Data sources reference +* [Tables](/tables/index.md) - Database tables reference diff --git a/open-knowledge-format/assets/example-bundle/tables/customers.md b/open-knowledge-format/assets/example-bundle/tables/customers.md new file mode 100644 index 0000000..d1a9c6e --- /dev/null +++ b/open-knowledge-format/assets/example-bundle/tables/customers.md @@ -0,0 +1,20 @@ +--- +type: BigQuery Table +title: Customers +description: One row per customer account. +tags: [sales, customer-data] +timestamp: 2026-06-18T00:00:00Z +--- + +# Schema + +| Column | Type | Description | +|--------|------|-------------| +| `customer_id` | STRING | Unique customer identifier. | +| `name` | STRING | Customer display name. | +| `email` | STRING | Customer email address. | +| `created_at` | TIMESTAMP | Account creation timestamp. | + +# Relationships + +Each customer has zero or more [orders](/tables/orders.md), linked by `customer_id`. diff --git a/open-knowledge-format/assets/example-bundle/tables/index.md b/open-knowledge-format/assets/example-bundle/tables/index.md new file mode 100644 index 0000000..9875659 --- /dev/null +++ b/open-knowledge-format/assets/example-bundle/tables/index.md @@ -0,0 +1,4 @@ +# Tables + +* [Customers](customers.md) - Customer profile data +* [Orders](orders.md) - Completed customer orders diff --git a/open-knowledge-format/assets/example-bundle/tables/orders.md b/open-knowledge-format/assets/example-bundle/tables/orders.md new file mode 100644 index 0000000..62941d0 --- /dev/null +++ b/open-knowledge-format/assets/example-bundle/tables/orders.md @@ -0,0 +1,18 @@ +--- +type: BigQuery Table +title: Orders +description: One row per completed customer order. +tags: [sales, orders] +timestamp: 2026-06-18T00:00:00Z +--- + +# Schema + +| Column | Type | Description | +|--------|------|-------------| +| `order_id` | STRING | Globally unique order identifier. | +| `customer_id` | STRING | Foreign key to [customers](/tables/customers.md). | +| `total_usd` | NUMERIC | Order total in US dollars. | +| `placed_at` | TIMESTAMP | When the customer submitted the order. | + +Part of the [sales dataset](/datasets/sales.md). diff --git a/open-knowledge-format/references/bundle-architecture.md b/open-knowledge-format/references/bundle-architecture.md new file mode 100644 index 0000000..e50f028 --- /dev/null +++ b/open-knowledge-format/references/bundle-architecture.md @@ -0,0 +1,145 @@ +# OKF Bundle Architecture + +How to structure, organize, and distribute OKF knowledge bundles effectively. + +## Bundle Distribution + +A bundle MAY be distributed as: + +- **A git repository** (recommended) — provides history, attribution, diffs +- **A tarball or zip archive** of the directory +- **A subdirectory** within a larger repository + +## Directory Organization Strategies + +### Flat structure (small bundles) + +For fewer than 10 concepts, a flat directory works: + +``` +bundle/ +├── index.md +├── customers.md +├── orders.md +├── weekly-active-users.md +└── incident-response.md +``` + +### Hierarchical structure (medium bundles) + +Group by domain or type for bundles with 10-100 concepts: + +``` +bundle/ +├── index.md +├── tables/ +│ ├── index.md +│ ├── customers.md +│ ├── orders.md +│ └── products.md +├── metrics/ +│ ├── index.md +│ ├── wau.md +│ └── revenue.md +└── playbooks/ + ├── index.md + ├── incident-response.md + └── data-freshness-alert.md +``` + +### Deeply nested structure (large bundles) + +For 100+ concepts, use subdirectories that mirror organizational or system boundaries: + +``` +bundle/ +├── index.md +├── log.md +├── sales/ +│ ├── index.md +│ ├── tables/ +│ │ ├── orders.md +│ │ └── customers.md +│ └── metrics/ +│ └── revenue.md +└── marketing/ + ├── index.md + ├── tables/ + │ └── campaigns.md + └── metrics/ + └── cac.md +``` + +## Cross-linking Patterns + +### Linking to a specific concept + +```markdown +See the [customers table](/tables/customers.md) for the join key. +``` + +### Linking to a subdirectory index + +```markdown +Browse available [metrics](/metrics/index.md). +``` + +### Linking to an external resource + +```markdown +Defined in the [OpenAPI spec](https://example.com/openapi.yaml). +``` + +### Linking from a dataset to its constituent tables + +```markdown +The sales dataset contains [orders](/tables/orders.md) and [customers](/tables/customers.md). +``` + +## Index File Design + +An `index.md` provides progressive disclosure — letting an agent or human see what's available without opening every document. + +### Root index pattern + +```markdown +# Knowledge Base + +* [Tables](/tables/index.md) - Database tables reference +* [Metrics](/metrics/index.md) - Business metrics definitions +* [Playbooks](/playbooks/index.md) - Incident response and operational guides +``` + +### Subdirectory index pattern + +```markdown +# Tables + +* [Customers](customers.md) - Customer profile data +* [Orders](orders.md) - Completed customer orders +* [Products](products.md) - Product catalog +``` + +## Log File Conventions + +A `log.md` records history at any level of the hierarchy: + +```markdown +# Change Log + +## 2026-06-18 +* **Creation**: Established the sales dataset documentation. +* **Update**: Added revenue metrics with cross-links to orders table. + +## 2026-06-15 +* **Initialization**: Created bundle structure and index files. +``` + +## Tips for Agent-Friendly Bundles + +1. **Always include `index.md` at the bundle root.** Agents use it as an entry point for progressive loading. +2. **Prefer absolute (bundle-relative) links** starting with `/`. They survive document moves within subdirectories. +3. **Use consistent `type` values across your bundle.** While the spec doesn't require a registry, consistency helps agents route and filter. +4. **Write descriptions for every concept.** These are what `index.md` entries and search snippets use. +5. **Bundle complementary domains separately.** A bundle about "sales data" and one about "incident response" are better as separate bundles than a single flat one. +6. **Tag liberally.** Tags are the primary cross-cutting categorization mechanism. They enable agent filtering without directory reorganization. diff --git a/open-knowledge-format/references/spec-summary.md b/open-knowledge-format/references/spec-summary.md new file mode 100644 index 0000000..914deca --- /dev/null +++ b/open-knowledge-format/references/spec-summary.md @@ -0,0 +1,165 @@ +# OKF v0.1 — Specification Summary + +The full specification lives at [GoogleCloudPlatform/knowledge-catalog/okf/SPEC.md](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md). This reference covers the key structural rules, conformance criteria, and design decisions. + +## Goals + +1. Define a universal format that **enrichment agents** can write into +2. Inform how **consumption agents** should read and traverse it +3. Facilitate **exchange** of knowledge across systems and organizations +4. Standardize the small number of **required** fields for meaningful consumption + +## Non-goals + +- Defining a fixed taxonomy of concept types +- Prescribing storage, serving, or query infrastructure +- Replacing domain-specific schemas (Avro, Protobuf, OpenAPI) — OKF references them, does not subsume them + +## Terminology + +| Term | Definition | +|------|------------| +| **Knowledge Bundle** | Self-contained, hierarchical collection of knowledge documents. The unit of distribution. | +| **Concept** | A single unit of knowledge within a bundle. One markdown document. | +| **Concept ID** | The path of the concept's file within the bundle, with `.md` suffix removed. E.g. `tables/users.md` has ID `tables/users`. | +| **Frontmatter** | YAML metadata block delimited by `---` at the top of a markdown file | +| **Body** | Everything in the file after the frontmatter | +| **Link** | Standard markdown link from one concept to another | +| **Citation** | A link from a concept to an external source supporting a claim | + +## Bundle Structure Rules + +``` +bundle/ +├── index.md # Reserved — directory listing (optional) +├── log.md # Reserved — update history (optional) +├── .md # Any other .md file is a concept +└── / + ├── index.md + └── … +``` + +### Reserved filenames + +| Filename | Purpose | +|----------|---------| +| `index.md` | Directory listing. Optional. No frontmatter (except at bundle root for `okf_version`). | +| `log.md` | Update history. Optional. ISO 8601 date headings, newest first. | + +All other `.md` files are concept documents. + +## Frontmatter Specification + +```yaml +--- +type: # REQUIRED +title: +description: +resource: +tags: [, …] # Optional +timestamp: # Optional +--- +``` + +### `type` field — REQUIRED + +A short string identifying the kind of concept. Examples: `BigQuery Table`, `BigQuery Dataset`, `API Endpoint`, `Metric`, `Playbook`, `Reference`. + +- Types are NOT registered centrally +- Producers SHOULD pick descriptive, self-explanatory values +- Consumers MUST tolerate unknown types gracefully (treat as generic concept) + +### Recommended fields (priority order) + +1. `title` — Human-readable display name. If omitted, consumers may derive from filename +2. `description` — Single sentence summary. Used by index generators, search snippets, previews +3. `resource` — URI identifying the underlying asset. Absent for abstract ideas +4. `tags` — YAML list of short strings for cross-cutting categorization +5. `timestamp` — ISO 8601 datetime of last meaningful change + +### Extensions + +Producers MAY include any additional keys. Consumers SHOULD preserve unknown keys on round-trip and SHOULD NOT reject unrecognized fields. + +## Conventional Body Sections + +| Heading | Purpose | +|---------|---------| +| `# Schema` | Structured column/field descriptions | +| `# Examples` | Usage examples, often fenced code blocks | +| `# Citations` | External sources backing claims | + +These are **conventions**, not requirements. Any markdown content is valid. + +## Cross-linking Rules + +Two forms of links: + +1. **Absolute (bundle-relative):** Start with `/`, interpreted relative to bundle root. Recommended form. +2. **Relative:** Standard markdown relative paths. + +A link from concept A to concept B asserts a relationship. The specific kind (parent/child, references, joins-with, depends-on) is conveyed by surrounding prose, not by the link itself. Graph consumers treat all links as directed edges of an untyped relationship. + +**Consumers MUST tolerate broken links** — a link whose target does not exist in the bundle is not malformed; it may represent not-yet-written knowledge. + +## Index Files (`index.md`) + +May appear in any directory. Contains no frontmatter (except optionally at bundle root): + +```markdown +# Section Heading + +* [Title 1](relative-url-1) - short description +* [Title 2](relative-url-2) - short description +``` + +Entries SHOULD include the `description` from the linked concept's frontmatter. Producers MAY generate `index.md` automatically; consumers MAY synthesize one on the fly. + +## Log Files (`log.md`) + +Flat list of date-grouped entries, newest first: + +```markdown +## 2026-05-22 +* **Update**: Added [Customer Metrics](/tables/customer-metrics.md). +* **Creation**: Established the [Playbook](/playbooks/dataplex.md). +``` + +Date headings MUST use ISO 8601 `YYYY-MM-DD` form. Leading bold word (`**Update**`, `**Creation**`, `**Deprecation**`) is a convention, not a requirement. + +## Citations + +Sources listed under `# Citations` at the bottom of a document, numbered: + +```markdown +# Citations + +[1] [Source title](https://...) +[2] [Source title](path/to/reference.md) +``` + +Citation links may be absolute URLs, bundle-relative paths, or paths into a `references/` subdirectory. + +## Conformance Criteria + +A bundle is **conformant** with OKF v0.1 if: + +1. Every non-reserved `.md` file in the tree contains a parseable YAML frontmatter block +2. Every frontmatter block contains a non-empty `type` field +3. Every reserved filename follows its defined structure when present + +Consumers MUST NOT reject a bundle because of: +- Missing optional frontmatter fields +- Unknown `type` values +- Unknown additional frontmatter keys +- Broken cross-links +- Missing `index.md` files + +## Versioning + +Format: `.` + +- **Minor bump:** Backward-compatible additions (new optional fields, new conventional headings) +- **Major bump:** Breaking changes (renaming required fields, changing reserved filenames) + +Bundles MAY declare their target version via `okf_version: "0.1"` in a bundle-root `index.md` frontmatter block (the only place frontmatter is permitted in `index.md`). Consumers that don't understand the declared version SHOULD attempt best-effort consumption. diff --git a/open-knowledge-format/references/use-cases.md b/open-knowledge-format/references/use-cases.md new file mode 100644 index 0000000..291319f --- /dev/null +++ b/open-knowledge-format/references/use-cases.md @@ -0,0 +1,78 @@ +# OKF — Real-World Use Cases and Adoption + +## The Problem OKF Solves + +Internal organizational knowledge is scattered across heterogeneous surfaces: + +- Metadata catalogs with proprietary APIs +- Wikis, third-party systems, shared drives +- Code comments, docstrings, notebook cells +- The heads of a few senior engineers + +When an AI agent needs to answer "How do I compute weekly active users from our event stream?" it has to assemble the answer from incompatible sources. Every agent builder reinvents context assembly; every catalog vendor reinvents data models. + +OKF provides a **format-level** solution — not another service, not another SDK. Anyone can produce OKF, anyone can consume it, and it survives moving between systems. + +## Google's Reference Implementation + +Google published a proof-of-concept in the same repository as the spec: + +- **Enrichment agent** — Built on the Google Agent Development Kit (ADK) with Gemini. Ingests BigQuery metadata and emits OKF bundles in two passes: a BQ pass (metadata extraction) and a web pass (LLM-driven crawling of documentation URLs for enrichment). +- **Interactive visualizer** — `viz.html`, a self-contained HTML file using Cytoscape.js that renders any OKF bundle as a force-directed graph with search, type filtering, and backlinks. + +Three sample bundles are checked into the repo: + +| Bundle | Source | Concepts | +|--------|--------|----------| +| `bundles/ga4/` | GA4 e-commerce dataset | BigQuery tables, metrics, dimensions | +| `bundles/stackoverflow/` | Stack Overflow public dataset | Schema references, cross-table joins | +| `bundles/crypto_bitcoin/` | Bitcoin blocks/transactions | Tables, foreign-key relationships | + +## Third-Party Adoption + +### Rust implementation (W4G1/okf) + +A pure-Rust, zero-dependency implementation of OKF v0.1 — demonstrates the spec is implementable without any Google-specific tooling. + +### Suganthan Mohanadasan + +Published his entire blog as an OKF bundle at `suganthan.com/okf/`. Uses it to make his writing directly consumable by AI agents. Agents start at `suganthan.com/okf/index.md` and navigate through linked concept files. + +### Marie Haynes + +Created an OKF bundle from her traffic-drop assessment methodology using Antigravity. Produced a graph visualization showing how extracted concepts relate. Tested querying it with Gemini 3 Flash as the consumption agent. + +## Predicted Use Cases + +### Internal knowledge management + +Organizations maintain OKF bundles alongside code repositories. Agents read the bundle to understand table schemas, metric definitions, and operational runbooks before performing tasks. Updates go through normal git workflows — PRs, reviews, merges. + +### Agent-to-agent knowledge exchange + +An agent working on a data pipeline produces an OKF bundle documenting the tables it creates and their semantics. A downstream agent consumes that bundle to understand how to query those tables. No API integration needed — just files in a shared repo. + +### Selling expert knowledge as bundles + +Professionals (lawyers, accountants, SEOs, consultants) package their proprietary processes as OKF bundles. Customers purchase and integrate them into their own agent systems. The format creates a marketplace for structured expert knowledge. + +### Cross-organizational data sharing + +Companies share OKF bundles with partners to document shared data schemas, business rules, and metrics. The format survives movement between organizations because it's just files. + +## Related Patterns + +OKF formalizes patterns already emerging in the ecosystem: + +- **Obsidian vaults** wired to coding agents +- **AGENTS.md / CLAUDE.md** convention files in repositories +- **LLM wiki repos** (Karpathy's pattern) — directories of `index.md` and `log.md` artifacts +- **Metadata-as-code** within data platform teams + +Each pattern independently solved the same problem; OKF standardizes the interoperability surface. + +## When Not to Use OKF + +- **Real-time query serving** — OKF is designed for curated knowledge, not sub-millisecond lookups. Use a vector DB or cache for serving. +- **High-volume transactional metadata** — OKF bundles are human-readable and git-friendly, but not designed for 100K+ concept updates per minute. +- **Replacing domain schemas** — Don't put Protobuf/ Avro/OpenAPI definitions in OKF. OKF *references* those schemas — it describes what they mean, not their binary wire format. diff --git a/open-knowledge-format/scripts/okf-bundle-validate.py b/open-knowledge-format/scripts/okf-bundle-validate.py new file mode 100644 index 0000000..2e07848 --- /dev/null +++ b/open-knowledge-format/scripts/okf-bundle-validate.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +""" +okf-bundle-validate.py — Validate an OKF v0.1 knowledge bundle. + +Checks: + 1. Every non-reserved .md file has parseable YAML frontmatter + 2. Every frontmatter block has a non-empty "type" field + 3. Reserved filenames (index.md, log.md) follow their defined structure + 4. Cross-links point to existing concept files (soft warning) + 5. No filename collisions between concept IDs + +Usage: + python3 okf-bundle-validate.py + python3 okf-bundle-validate.py --fix-missing-types + python3 okf-bundle-validate.py --json +""" + +import os +import sys +import json +import re +import argparse +from pathlib import Path + +RESERVED = {"index.md", "log.md"} +DATE_HEADING_RE = re.compile(r"^##\s+\d{4}-\d{2}-\d{2}") +LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") +MD_LINK_RE = re.compile(r"\]\(((?:/|\./|\.\./)?[^)]+\.md)\)") + + +class ValidationResult: + def __init__(self): + self.errors = [] + self.warnings = [] + self.info = [] + self.stats = { + "concept_files": 0, + "reserved_files": 0, + "cross_links": 0, + "broken_links": 0, + } + + def error(self, path, msg): + self.errors.append({"path": str(path), "message": msg}) + + def warning(self, path, msg): + self.warnings.append({"path": str(path), "message": msg}) + + def add_info(self, msg): + self.info.append(msg) + + @property + def passed(self): + return len(self.errors) == 0 + + def to_dict(self): + return { + "passed": self.passed, + "errors": self.errors, + "warnings": self.warnings, + "info": self.info, + "stats": self.stats, + } + + def summary(self): + parts = [] + if self.errors: + parts.append(f"FAILED: {len(self.errors)} error(s)") + else: + parts.append("PASSED") + parts.append(f"{self.stats['concept_files']} concepts") + parts.append(f"{self.stats['reserved_files']} reserved files") + parts.append(f"{self.stats['cross_links']} cross-links") + if self.stats["broken_links"]: + parts.append(f"{self.stats['broken_links']} broken link(s)") + return ", ".join(parts) + + +def parse_frontmatter(content): + """Parse YAML frontmatter from markdown content. Returns (fields, body, error).""" + if not content.startswith("---"): + return None, content, "File does not start with frontmatter delimiter" + + # Find closing --- + end_idx = content.find("\n---", 3) + if end_idx == -1: + return None, content, "Frontmatter not closed (missing closing ---)" + + yaml_block = content[3:end_idx].strip() + body = content[end_idx + 4:].strip() + + if not yaml_block: + return {}, body, None + + # Simple YAML parser for the subset OKF uses + fields = {} + current_key = None + list_mode = False + + for line in yaml_block.split("\n"): + # Skip blank lines and comments + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + + # Tag list continuation + if list_mode and stripped.startswith("- "): + if isinstance(fields.get(current_key), list): + fields[current_key].append(stripped[2:]) + continue + else: + list_mode = False + + # Key-value pair + if ":" in stripped and not stripped.startswith("- "): + colon_idx = stripped.index(":") + key = stripped[:colon_idx].strip() + value = stripped[colon_idx + 1:].strip() + current_key = key + + if not value or value == "|" or value == ">": + fields[key] = "" + elif value.startswith("[") and value.endswith("]"): + # Inline list + inner = value[1:-1] + items = [item.strip().strip("'\"") for item in inner.split(",") if item.strip()] + fields[key] = items + elif value.startswith("- "): + list_mode = True + fields[key] = [value[2:]] + elif value.lower() == "true": + fields[key] = True + elif value.lower() == "false": + fields[key] = False + else: + # Remove surrounding quotes + if (value.startswith("'") and value.endswith("'")) or \ + (value.startswith('"') and value.endswith('"')): + value = value[1:-1] + fields[key] = value + + return fields, body, None + + +def collect_md_files(bundle_path): + """Collect all .md files in the bundle directory.""" + md_files = [] + for root, dirs, files in os.walk(bundle_path): + # Skip hidden dirs + dirs[:] = [d for d in dirs if not d.startswith(".")] + for f in files: + if f.endswith(".md"): + abs_path = Path(root) / f + rel_path = abs_path.relative_to(bundle_path) + md_files.append(rel_path) + return sorted(md_files) + + +def check_index_file(path, content, result): + """Check an index.md file follows OKF conventions.""" + lines = content.strip().split("\n") + + # Check for frontmatter (should NOT have it, except at bundle root) + if content.startswith("---"): + # Only allowed at bundle root for okf_version + if str(path) != "index.md": + result.warning(path, "index.md in subdirectory contains frontmatter (should be frontmatter-free)") + else: + # Has frontmatter - check it only contains okf_version + end_idx = content.find("\n---", 3) + if end_idx != -1: + yaml_block = content[3:end_idx].strip() + if yaml_block and "okf_version" not in yaml_block: + result.warning(path, "Root index.md frontmatter should only contain okf_version") + + +def check_log_file(path, content, result): + """Check a log.md file follows OKF conventions.""" + lines = content.strip().split("\n") + date_headings = 0 + + for line in lines: + if DATE_HEADING_RE.match(line.strip()): + date_headings += 1 + + if date_headings == 0: + result.warning(path, "log.md has no ISO 8601 date headings (## YYYY-MM-DD)") + + +def validate_bundle(bundle_path, fix_missing_types=False): + """Validate an OKF v0.1 bundle directory.""" + result = ValidationResult() + bundle_path = Path(bundle_path).resolve() + + if not bundle_path.is_dir(): + result.error(bundle_path, "Not a directory") + return result + + result.add_info(f"Bundle path: {bundle_path}") + + md_files = collect_md_files(bundle_path) + if not md_files: + result.error(bundle_path, "No markdown files found in bundle") + return result + + # Build concept ID map + concept_ids = {} + for rel_path in md_files: + stem = str(rel_path.with_suffix("")) + concept_ids[stem] = rel_path + + # Validate each file + for rel_path in md_files: + abs_path = bundle_path / rel_path + filename = rel_path.name + + try: + content = abs_path.read_text(encoding="utf-8") + except Exception as e: + result.error(rel_path, f"Cannot read file: {e}") + continue + + is_reserved = filename in RESERVED + + if is_reserved: + result.stats["reserved_files"] += 1 + else: + result.stats["concept_files"] += 1 + + # Check frontmatter + fields, body, parse_err = parse_frontmatter(content) + + if parse_err: + if is_reserved: + result.add_info(f"Reserved file {rel_path} has no frontmatter (expected for {filename})") + else: + result.error(rel_path, parse_err) + continue + + # Check reserved file structure + if filename == "index.md" and is_reserved: + check_index_file(rel_path, content, result) + elif filename == "log.md" and is_reserved: + check_log_file(rel_path, content, result) + + if is_reserved: + continue + + # Check required 'type' field + if not fields or "type" not in fields or not fields.get("type"): + if fix_missing_types: + # We don't actually fix here — this is informational + result.error(rel_path, "Missing required 'type' field in frontmatter") + else: + result.error(rel_path, "Missing required 'type' field in frontmatter") + continue + + # Check cross-links + md_links = MD_LINK_RE.findall(body) if body else [] + for link in md_links: + result.stats["cross_links"] += 1 + + # Resolve relative links + if link.startswith("/"): + # Bundle-relative + target = link[1:].replace(".md", "") + elif link.startswith("./") or link.startswith("../"): + # Relative — resolve from file's directory + file_dir = str(rel_path.parent) if str(rel_path.parent) != "." else "" + target = os.path.normpath(os.path.join(file_dir, link)).replace(".md", "") + else: + # External URL or same-file anchor — skip + continue + + # Remove anchors + target = target.split("#")[0] + + if target and target not in concept_ids: + result.stats["broken_links"] += 1 + result.warning(rel_path, f"Broken link to '{target}.md' (concept ID: {target})") + + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Validate an OKF v0.1 knowledge bundle" + ) + parser.add_argument( + "bundle", + help="Path to the OKF bundle directory" + ) + parser.add_argument( + "--fix-missing-types", + action="store_true", + help="Flag missing type fields as errors (default: errors)" + ) + parser.add_argument( + "--json", + action="store_true", + help="Output results as JSON" + ) + + args = parser.parse_args() + + result = validate_bundle(args.bundle, fix_missing_types=args.fix_missing_types) + + if args.json: + print(json.dumps(result.to_dict(), indent=2)) + else: + print("\n" + "=" * 60) + print(f" OKF Bundle Validation: {result.summary()}") + print("=" * 60) + + if result.errors: + print(f"\n Errors ({len(result.errors)}):") + for err in result.errors: + print(f" ✗ {err['path']}: {err['message']}") + + if result.warnings: + print(f"\n Warnings ({len(result.warnings)}):") + for warn in result.warnings: + print(f" ⚠ {warn['path']}: {warn['message']}") + + if result.info: + print(f"\n Info:") + for msg in result.info: + print(f" ℹ {msg}") + + print() + + return 0 if result.passed else 1 + + +if __name__ == "__main__": + sys.exit(main())