diff --git a/OPERATIONS.md b/OPERATIONS.md new file mode 100644 index 0000000..325735a --- /dev/null +++ b/OPERATIONS.md @@ -0,0 +1,702 @@ +# Operation Vocabulary Reference + +This document provides a complete reference for all operations available in the goodpackagerepo pipeline system. + +## Overview + +The repository uses a **closed-world operations model**, meaning only explicitly allowed operations can be used in pipeline definitions. This ensures security, predictability, and static validation. + +## Operation Categories + +### Authentication Operations + +#### `auth.require_scopes` +Require specific authentication scopes for access. + +**Arguments:** +- `scopes` (array of strings) - Required scopes (e.g., `["read"]`, `["write"]`, `["admin"]`) + +**Example:** +```json +{ + "op": "auth.require_scopes", + "args": { + "scopes": ["write"] + } +} +``` + +--- + +### Parsing Operations + +#### `parse.path` +Parse URL path parameters into entity fields. + +**Arguments:** +- `entity` (string) - Entity type to parse into (e.g., `"artifact"`) + +**Example:** +```json +{ + "op": "parse.path", + "args": { + "entity": "artifact" + } +} +``` + +#### `parse.query` +Parse URL query parameters. + +**Arguments:** +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "parse.query", + "args": { + "out": "query_params" + } +} +``` + +#### `parse.json` +Parse JSON request body. + +**Arguments:** +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "parse.json", + "args": { + "out": "body" + } +} +``` + +--- + +### Normalization and Validation Operations + +#### `normalize.entity` +Normalize entity fields according to schema rules (trim, lowercase, replacements). + +**Arguments:** +- `entity` (string) - Entity type to normalize + +**Normalization Rules:** +- `trim` - Remove leading/trailing whitespace +- `lower` - Convert to lowercase +- `replace:X:Y` - Replace X with Y + +**Example:** +```json +{ + "op": "normalize.entity", + "args": { + "entity": "artifact" + } +} +``` + +#### `validate.entity` +Validate entity against schema constraints (regex patterns, required fields). + +**Arguments:** +- `entity` (string) - Entity type to validate + +**Example:** +```json +{ + "op": "validate.entity", + "args": { + "entity": "artifact" + } +} +``` + +#### `validate.json_schema` +Validate data against a JSON schema. + +**Arguments:** +- `schema` (object) - JSON schema definition +- `value` (any) - Value to validate + +**Example:** +```json +{ + "op": "validate.json_schema", + "args": { + "schema": { + "type": "object", + "required": ["name"], + "properties": { + "name": {"type": "string"} + } + }, + "value": "$body" + } +} +``` + +--- + +### Transaction Operations + +#### `txn.begin` +Begin a database transaction. + +**Arguments:** +- `isolation` (string) - Isolation level (`"serializable"`, `"repeatable_read"`, `"read_committed"`) + +**Example:** +```json +{ + "op": "txn.begin", + "args": { + "isolation": "serializable" + } +} +``` + +#### `txn.commit` +Commit the current transaction. + +**Arguments:** None + +**Example:** +```json +{ + "op": "txn.commit", + "args": {} +} +``` + +#### `txn.abort` +Abort the current transaction. + +**Arguments:** None + +**Example:** +```json +{ + "op": "txn.abort", + "args": {} +} +``` + +--- + +### Key-Value Store Operations + +#### `kv.get` +Get a value from the key-value store. + +**Arguments:** +- `doc` (string) - Document type (e.g., `"artifact_meta"`, `"tag_map"`) +- `key` (string) - Key template with variable interpolation +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "kv.get", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}/{version}/{variant}", + "out": "meta" + } +} +``` + +#### `kv.put` +Put a value into the key-value store. + +**Arguments:** +- `doc` (string) - Document type +- `key` (string) - Key template +- `value` (any) - Value to store + +**Example:** +```json +{ + "op": "kv.put", + "args": { + "doc": "tag_map", + "key": "tag/{namespace}/{name}/{tag}", + "value": { + "namespace": "{namespace}", + "tag": "{tag}", + "target": "{target_version}" + } + } +} +``` + +#### `kv.cas_put` +Compare-and-swap put - conditional write if absent or matches expected value. + +**Arguments:** +- `doc` (string) - Document type +- `key` (string) - Key template +- `value` (any) - Value to store +- `if_absent` (boolean) - Only write if key doesn't exist + +**Example:** +```json +{ + "op": "kv.cas_put", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}/{version}/{variant}", + "if_absent": true, + "value": "$metadata" + } +} +``` + +#### `kv.delete` +Delete a key from the key-value store. + +**Arguments:** +- `doc` (string) - Document type +- `key` (string) - Key template + +**Example:** +```json +{ + "op": "kv.delete", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}/{version}/{variant}" + } +} +``` + +--- + +### Blob Store Operations + +#### `blob.get` +Get a blob from the blob store. + +**Arguments:** +- `store` (string) - Blob store name (e.g., `"primary"`) +- `digest` (string) - Blob digest (content hash) +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "blob.get", + "args": { + "store": "primary", + "digest": "$meta.blob_digest", + "out": "blob" + } +} +``` + +#### `blob.put` +Put a blob into the blob store. + +**Arguments:** +- `store` (string) - Blob store name +- `from` (string) - Source (e.g., `"request.body"`, variable) +- `out` (string) - Output variable for digest +- `out_size` (string) - Output variable for size + +**Example:** +```json +{ + "op": "blob.put", + "args": { + "store": "primary", + "from": "request.body", + "out": "digest", + "out_size": "blob_size" + } +} +``` + +#### `blob.verify_digest` +Verify blob integrity by checking digest. + +**Arguments:** +- `digest` (string) - Digest to verify +- `algo` (string) - Hash algorithm (e.g., `"sha256"`) + +**Example:** +```json +{ + "op": "blob.verify_digest", + "args": { + "digest": "$digest", + "algo": "sha256" + } +} +``` + +--- + +### Index Operations + +#### `index.query` +Query an index. + +**Arguments:** +- `index` (string) - Index name +- `key` (object) - Query key fields +- `limit` (integer) - Maximum results +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "index.query", + "args": { + "index": "artifact_versions", + "key": { + "namespace": "{namespace}", + "name": "{name}" + }, + "limit": 10, + "out": "rows" + } +} +``` + +#### `index.upsert` +Insert or update an index entry. + +**Arguments:** +- `index` (string) - Index name +- `key` (object) - Index key fields +- `value` (object) - Value to store + +**Example:** +```json +{ + "op": "index.upsert", + "args": { + "index": "artifact_versions", + "key": { + "namespace": "{namespace}", + "name": "{name}" + }, + "value": { + "version": "{version}", + "variant": "{variant}" + } + } +} +``` + +#### `index.delete` +Delete from an index. + +**Arguments:** +- `index` (string) - Index name +- `key` (object) - Index key fields + +**Example:** +```json +{ + "op": "index.delete", + "args": { + "index": "artifact_versions", + "key": { + "namespace": "{namespace}", + "name": "{name}", + "version": "{version}" + } + } +} +``` + +--- + +### Cache Operations + +#### `cache.get` +Get a value from the cache. + +**Arguments:** +- `kind` (string) - Cache kind (`"response"`, `"blob"`) +- `key` (string) - Cache key +- `hit_out` (string) - Output variable for cache hit status +- `value_out` (string) - Output variable for cached value + +**Example:** +```json +{ + "op": "cache.get", + "args": { + "kind": "response", + "key": "blob_resp/{namespace}/{name}/{version}/{variant}", + "hit_out": "cache_hit", + "value_out": "cached_resp" + } +} +``` + +#### `cache.put` +Put a value into the cache. + +**Arguments:** +- `kind` (string) - Cache kind +- `key` (string) - Cache key +- `ttl_seconds` (integer) - Time to live +- `value` (any) - Value to cache + +**Example:** +```json +{ + "op": "cache.put", + "args": { + "kind": "response", + "key": "blob_resp/{namespace}/{name}/{version}/{variant}", + "ttl_seconds": 300, + "value": "$response_data" + } +} +``` + +--- + +### Proxy Operations + +#### `proxy.fetch` +Fetch from an upstream proxy. + +**Arguments:** +- `upstream` (string) - Upstream name +- `method` (string) - HTTP method +- `path` (string) - Request path +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "proxy.fetch", + "args": { + "upstream": "originA", + "method": "GET", + "path": "/v1/{namespace}/{name}/{version}/{variant}/blob", + "out": "up_resp" + } +} +``` + +--- + +### Response Operations + +#### `respond.json` +Return a JSON response. + +**Arguments:** +- `status` (integer) - HTTP status code +- `body` (object) - Response body +- `when` (object, optional) - Conditional execution + +**Example:** +```json +{ + "op": "respond.json", + "args": { + "status": 200, + "body": { + "ok": true, + "data": "$result" + } + } +} +``` + +#### `respond.bytes` +Return a binary response. + +**Arguments:** +- `status` (integer) - HTTP status code +- `body` (any) - Response body +- `headers` (object, optional) - Response headers +- `when` (object, optional) - Conditional execution + +**Example:** +```json +{ + "op": "respond.bytes", + "args": { + "status": 200, + "headers": { + "Content-Type": "application/octet-stream" + }, + "body": "$blob" + } +} +``` + +#### `respond.redirect` +Return a redirect response. + +**Arguments:** +- `status` (integer) - HTTP status code (301, 302, 307, 308) +- `location` (string) - Redirect URL +- `when` (object, optional) - Conditional execution + +**Example:** +```json +{ + "op": "respond.redirect", + "args": { + "status": 307, + "location": "/v1/{namespace}/{name}/{version}/{variant}/blob" + } +} +``` + +#### `respond.error` +Return an error response. + +**Arguments:** +- `status` (integer) - HTTP status code +- `code` (string) - Error code +- `message` (string) - Error message +- `when` (object, optional) - Conditional execution + +**Example:** +```json +{ + "op": "respond.error", + "args": { + "when": { + "is_null": "$meta" + }, + "status": 404, + "code": "NOT_FOUND", + "message": "Artifact not found" + } +} +``` + +--- + +### Event Operations + +#### `emit.event` +Emit an event to the event log for replication and auditing. + +**Arguments:** +- `type` (string) - Event type name +- `payload` (object) - Event payload + +**Example:** +```json +{ + "op": "emit.event", + "args": { + "type": "artifact.published", + "payload": { + "namespace": "{namespace}", + "name": "{name}", + "version": "{version}", + "at": "$now", + "by": "{principal.sub}" + } + } +} +``` + +--- + +### Utility Operations + +#### `time.now_iso8601` +Get the current time in ISO8601 format. + +**Arguments:** +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "time.now_iso8601", + "args": { + "out": "now" + } +} +``` + +#### `string.format` +Format strings with variable interpolation. + +**Arguments:** +- `template` (string) - String template +- `out` (string) - Output variable name + +**Example:** +```json +{ + "op": "string.format", + "args": { + "template": "{namespace}/{name}:{version}", + "out": "formatted" + } +} +``` + +--- + +## Variable Interpolation + +Operations support variable interpolation using: + +- `{field}` - Path/entity field (e.g., `{namespace}`, `{version}`) +- `$variable` - Runtime variable (e.g., `$digest`, `$body`) +- `{principal.sub}` - Principal field from JWT token + +## Conditional Execution + +Many operations support conditional execution via the `when` argument: + +```json +{ + "when": { + "equals": ["$var1", "$var2"], + "is_null": "$var", + "is_not_null": "$var", + "is_empty": "$list", + "not_in": ["$value", [1, 2, 3]] + } +} +``` + +## Pipeline Limits + +- Maximum operations per pipeline: 128 +- Maximum request body: 2GB +- Maximum JSON size: 10MB +- Maximum KV value size: 1MB +- Maximum CPU time per request: 200ms +- Maximum I/O operations per request: 5000 + +## Best Practices + +1. **Always use transactions** for operations that modify data (`kv.put`, `index.upsert`) +2. **Verify blob digests** after blob.put to ensure integrity +3. **Use caching** for read-heavy endpoints +4. **Emit events** for audit trail and replication +5. **Validate early** - parse, normalize, and validate before processing +6. **Check auth first** - require_scopes should be the first operation +7. **Handle errors gracefully** - use respond.error with appropriate status codes + +## See Also + +- `schema.json` - Complete schema definition +- `templates/` - Example pipeline templates +- API Routes documentation diff --git a/README.md b/README.md index 7c8e177..694a313 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,36 @@ npm install npm run dev ``` +## Seed Data and Templates + +### Load Example Data + +To populate your repository with example packages for testing: + +```bash +cd seed_data +pip install requests +python load_seed_data.py +``` + +This loads sample packages including: +- `acme/hello-world` - Multi-version example with multiple variants +- `example/webapp` - Web application containers +- `tools/cli-tool` - CLI tool example +- `libs/utility` - Library with prerelease versions + +### Templates + +The `templates/` directory contains reusable templates for: +- **Entity definitions** - Define new data models +- **API routes** - Create custom endpoints +- **Pipeline patterns** - Common operation sequences +- **Blob stores** - Configure storage backends +- **Auth scopes** - Define permission sets +- **Upstream proxies** - Configure external repositories + +See `templates/README.md` for the complete operation vocabulary and usage examples. + ## Documentation Complete documentation is available at `/docs` when running the application, including: @@ -54,6 +84,7 @@ Complete documentation is available at `/docs` when running the application, inc - CapRover Deployment Instructions - API Usage Examples - Schema Configuration +- Operation Vocabulary Reference ## Testing diff --git a/seed_data/README.md b/seed_data/README.md new file mode 100644 index 0000000..c60b74e --- /dev/null +++ b/seed_data/README.md @@ -0,0 +1,36 @@ +# Seed Data + +This directory contains example seed data for the goodpackagerepo system. Use this data to: + +- Test the repository functionality +- Demonstrate features to users +- Provide working examples + +## Contents + +- `example_packages.json` - Sample package metadata for testing +- `load_seed_data.py` - Script to load seed data into the repository +- `sample_blobs/` - Directory containing sample blob files to upload + +## Usage + +To load seed data into your repository: + +```bash +cd seed_data +python load_seed_data.py +``` + +This will: +1. Create sample artifacts in various namespaces +2. Tag them appropriately +3. Demonstrate the full artifact lifecycle + +## Example Packages + +The seed data includes: + +- **acme/hello-world** (v1.0.0, v1.1.0, v2.0.0) - Simple hello world packages +- **example/webapp** (v0.1.0, v0.2.0) - Web application example +- **tools/cli-tool** (v3.0.0) - CLI tool example +- **libs/utility** (v1.0.0-beta, v1.0.0) - Library with prerelease versions diff --git a/seed_data/example_packages.json b/seed_data/example_packages.json new file mode 100644 index 0000000..909cb3b --- /dev/null +++ b/seed_data/example_packages.json @@ -0,0 +1,158 @@ +{ + "packages": [ + { + "namespace": "acme", + "name": "hello-world", + "version": "1.0.0", + "variant": "linux-amd64", + "description": "Simple hello world application for Linux AMD64", + "content": "Hello World v1.0.0 - This is a sample package blob content", + "labels": { + "platform": "linux", + "arch": "amd64", + "language": "go" + } + }, + { + "namespace": "acme", + "name": "hello-world", + "version": "1.1.0", + "variant": "linux-amd64", + "description": "Hello world application v1.1.0 with bug fixes", + "content": "Hello World v1.1.0 - Updated with bug fixes and improvements", + "labels": { + "platform": "linux", + "arch": "amd64", + "language": "go" + } + }, + { + "namespace": "acme", + "name": "hello-world", + "version": "2.0.0", + "variant": "linux-amd64", + "description": "Hello world application v2.0.0 - major rewrite", + "content": "Hello World v2.0.0 - Complete rewrite with new features", + "labels": { + "platform": "linux", + "arch": "amd64", + "language": "rust" + } + }, + { + "namespace": "acme", + "name": "hello-world", + "version": "2.0.0", + "variant": "darwin-arm64", + "description": "Hello world application v2.0.0 for macOS ARM64", + "content": "Hello World v2.0.0 - macOS ARM64 version", + "labels": { + "platform": "darwin", + "arch": "arm64", + "language": "rust" + } + }, + { + "namespace": "example", + "name": "webapp", + "version": "0.1.0", + "variant": "container", + "description": "Simple web application container", + "content": "WebApp v0.1.0 - Container image with Node.js application", + "labels": { + "type": "container", + "runtime": "nodejs", + "framework": "express" + } + }, + { + "namespace": "example", + "name": "webapp", + "version": "0.2.0", + "variant": "container", + "description": "Web application v0.2.0 with new features", + "content": "WebApp v0.2.0 - Added authentication and database support", + "labels": { + "type": "container", + "runtime": "nodejs", + "framework": "express", + "features": "auth,database" + } + }, + { + "namespace": "tools", + "name": "cli-tool", + "version": "3.0.0", + "variant": "universal", + "description": "Universal CLI tool for multiple platforms", + "content": "CLI Tool v3.0.0 - Cross-platform command line utility", + "labels": { + "type": "cli", + "platform": "universal" + } + }, + { + "namespace": "libs", + "name": "utility", + "version": "1.0.0-beta", + "variant": "npm", + "description": "Utility library beta version", + "content": "Utility Library v1.0.0-beta - NPM package for JavaScript utilities", + "labels": { + "type": "library", + "ecosystem": "npm", + "language": "javascript" + } + }, + { + "namespace": "libs", + "name": "utility", + "version": "1.0.0", + "variant": "npm", + "description": "Utility library stable release", + "content": "Utility Library v1.0.0 - Stable NPM package for JavaScript utilities", + "labels": { + "type": "library", + "ecosystem": "npm", + "language": "javascript" + } + } + ], + "tags": [ + { + "namespace": "acme", + "name": "hello-world", + "tag": "latest", + "target_version": "2.0.0", + "target_variant": "linux-amd64" + }, + { + "namespace": "acme", + "name": "hello-world", + "tag": "stable", + "target_version": "1.1.0", + "target_variant": "linux-amd64" + }, + { + "namespace": "example", + "name": "webapp", + "tag": "latest", + "target_version": "0.2.0", + "target_variant": "container" + }, + { + "namespace": "tools", + "name": "cli-tool", + "tag": "latest", + "target_version": "3.0.0", + "target_variant": "universal" + }, + { + "namespace": "libs", + "name": "utility", + "tag": "latest", + "target_version": "1.0.0", + "target_variant": "npm" + } + ] +} diff --git a/seed_data/load_seed_data.py b/seed_data/load_seed_data.py new file mode 100755 index 0000000..d04ffa4 --- /dev/null +++ b/seed_data/load_seed_data.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +Seed Data Loader for goodpackagerepo + +This script loads example packages and tags into the repository +for testing and demonstration purposes. +""" + +import json +import os +import sys +import requests +from pathlib import Path +from typing import Dict, Any + +# Configuration +BACKEND_URL = os.environ.get("BACKEND_URL", "http://localhost:5000") +ADMIN_USERNAME = os.environ.get("ADMIN_USERNAME", "admin") +ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "admin") + + +def login(username: str, password: str) -> str: + """Login and get JWT token.""" + response = requests.post( + f"{BACKEND_URL}/auth/login", + json={"username": username, "password": password} + ) + + if response.status_code != 200: + print(f"❌ Login failed: {response.status_code}") + print(response.text) + sys.exit(1) + + data = response.json() + print(f"✅ Logged in as {username}") + return data["token"] + + +def publish_package(token: str, package: Dict[str, Any]) -> None: + """Publish a package to the repository.""" + namespace = package["namespace"] + name = package["name"] + version = package["version"] + variant = package["variant"] + content = package["content"].encode("utf-8") + + url = f"{BACKEND_URL}/v1/{namespace}/{name}/{version}/{variant}/blob" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/octet-stream" + } + + response = requests.put(url, headers=headers, data=content) + + if response.status_code == 201: + data = response.json() + print(f"✅ Published {namespace}/{name}:{version}@{variant} (digest: {data['digest'][:16]}...)") + elif response.status_code == 409: + print(f"⚠️ Package {namespace}/{name}:{version}@{variant} already exists, skipping") + else: + print(f"❌ Failed to publish {namespace}/{name}:{version}@{variant}: {response.status_code}") + print(response.text) + + +def set_tag(token: str, tag_info: Dict[str, Any]) -> None: + """Set a tag for a package.""" + namespace = tag_info["namespace"] + name = tag_info["name"] + tag = tag_info["tag"] + target_version = tag_info["target_version"] + target_variant = tag_info["target_variant"] + + url = f"{BACKEND_URL}/v1/{namespace}/{name}/tags/{tag}" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + + response = requests.put( + url, + headers=headers, + json={ + "target_version": target_version, + "target_variant": target_variant + } + ) + + if response.status_code == 200: + print(f"✅ Set tag {namespace}/{name}:{tag} -> {target_version}@{target_variant}") + else: + print(f"❌ Failed to set tag {namespace}/{name}:{tag}: {response.status_code}") + print(response.text) + + +def main(): + """Main function to load seed data.""" + print("=" * 60) + print("📦 goodpackagerepo Seed Data Loader") + print("=" * 60) + print() + + # Check if backend is reachable + try: + response = requests.get(f"{BACKEND_URL}/health", timeout=5) + if response.status_code == 200: + print(f"✅ Backend is reachable at {BACKEND_URL}") + else: + print(f"⚠️ Backend returned status {response.status_code}") + except Exception as e: + print(f"❌ Cannot reach backend at {BACKEND_URL}: {e}") + print(" Make sure the backend is running first.") + sys.exit(1) + + print() + + # Login + token = login(ADMIN_USERNAME, ADMIN_PASSWORD) + print() + + # Load seed data + seed_file = Path(__file__).parent / "example_packages.json" + with open(seed_file) as f: + seed_data = json.load(f) + + # Publish packages + print("Publishing packages...") + print("-" * 60) + for package in seed_data["packages"]: + publish_package(token, package) + + print() + print("Setting tags...") + print("-" * 60) + for tag in seed_data["tags"]: + set_tag(token, tag) + + print() + print("=" * 60) + print("✅ Seed data loaded successfully!") + print("=" * 60) + print() + print("You can now:") + print(f" - List packages: curl {BACKEND_URL}/v1///versions") + print(f" - Get latest: curl {BACKEND_URL}/v1///latest") + print(f" - Download: curl {BACKEND_URL}/v1/////blob") + print() + + +if __name__ == "__main__": + main() diff --git a/templates/README.md b/templates/README.md new file mode 100644 index 0000000..f4226e2 --- /dev/null +++ b/templates/README.md @@ -0,0 +1,90 @@ +# Templates + +This directory contains template files for creating new entities, routes, and configurations in the goodpackagerepo system. + +## Contents + +- `entity_template.json` - Template for defining new entity types +- `route_template.json` - Template for creating new API routes +- `pipeline_template.json` - Template for building operation pipelines +- `blob_store_template.json` - Template for configuring blob stores +- `auth_scope_template.json` - Template for defining authentication scopes +- `upstream_template.json` - Template for configuring upstream repositories + +## Operation Vocabulary + +The repository supports a closed-world set of operations that can be used in pipeline definitions: + +### Authentication Operations +- `auth.require_scopes` - Require specific scopes for access + +### Parsing Operations +- `parse.path` - Parse path parameters into entity fields +- `parse.query` - Parse query parameters +- `parse.json` - Parse JSON request body + +### Normalization and Validation +- `normalize.entity` - Normalize entity fields (trim, lowercase, etc.) +- `validate.entity` - Validate entity against constraints +- `validate.json_schema` - Validate data against JSON schema + +### Transaction Operations +- `txn.begin` - Begin a transaction +- `txn.commit` - Commit a transaction +- `txn.abort` - Abort a transaction + +### Key-Value Store Operations +- `kv.get` - Get value from KV store +- `kv.put` - Put value into KV store +- `kv.cas_put` - Compare-and-swap put (conditional) +- `kv.delete` - Delete from KV store + +### Blob Store Operations +- `blob.get` - Get blob from store +- `blob.put` - Put blob into store +- `blob.verify_digest` - Verify blob integrity + +### Index Operations +- `index.query` - Query an index +- `index.upsert` - Insert or update index entry +- `index.delete` - Delete from index + +### Cache Operations +- `cache.get` - Get from cache +- `cache.put` - Put into cache + +### Proxy Operations +- `proxy.fetch` - Fetch from upstream proxy + +### Response Operations +- `respond.json` - Return JSON response +- `respond.bytes` - Return binary response +- `respond.redirect` - Return redirect response +- `respond.error` - Return error response + +### Event Operations +- `emit.event` - Emit an event to the event log + +### Utility Operations +- `time.now_iso8601` - Get current time in ISO8601 format +- `string.format` - Format strings with variables + +## Usage + +Copy a template file and customize it for your needs. Templates use placeholders that should be replaced: + +- `{namespace}` - Package namespace +- `{name}` - Package name +- `{version}` - Package version +- `{variant}` - Package variant +- `$variable` - Runtime variable from pipeline execution + +## Example + +To create a new route based on a template: + +```bash +cp templates/route_template.json my_route.json +# Edit my_route.json with your route definition +# Use the admin API to add the route to the system +``` diff --git a/templates/auth_scope_template.json b/templates/auth_scope_template.json new file mode 100644 index 0000000..420e4f7 --- /dev/null +++ b/templates/auth_scope_template.json @@ -0,0 +1,25 @@ +{ + "name": "custom_scope", + "description": "Custom authentication scope", + "actions": [ + "blob.get", + "blob.put", + "kv.get", + "kv.put", + "index.query" + ], + "examples": [ + { + "use_case": "Read-only scope", + "actions": ["blob.get", "kv.get", "index.query"] + }, + { + "use_case": "Write-only scope", + "actions": ["blob.put", "kv.put", "index.upsert"] + }, + { + "use_case": "Admin scope", + "actions": ["*"] + } + ] +} diff --git a/templates/blob_store_template.json b/templates/blob_store_template.json new file mode 100644 index 0000000..f63d492 --- /dev/null +++ b/templates/blob_store_template.json @@ -0,0 +1,22 @@ +{ + "name": "my_blob_store", + "kind": "filesystem", + "root": "/data/blobs", + "addressing": { + "mode": "content_addressed", + "digest": "sha256", + "path_template": "sha256/{digest:0:2}/{digest:2:2}/{digest}", + "description": "Content-addressed storage with 2-level directory sharding" + }, + "limits": { + "max_blob_bytes": 2147483648, + "min_blob_bytes": 0, + "description": "Max 2GB per blob" + }, + "features": { + "compression": false, + "encryption": false, + "deduplication": true, + "description": "Deduplication via content addressing" + } +} diff --git a/templates/entity_template.json b/templates/entity_template.json new file mode 100644 index 0000000..89d357f --- /dev/null +++ b/templates/entity_template.json @@ -0,0 +1,38 @@ +{ + "name": "my_entity", + "description": "Description of what this entity represents", + "fields": { + "field1": { + "type": "string", + "optional": false, + "normalize": ["trim", "lower"], + "description": "First field - normalized to lowercase" + }, + "field2": { + "type": "string", + "optional": true, + "normalize": ["trim"], + "description": "Optional second field" + }, + "field3": { + "type": "string", + "optional": false, + "normalize": ["trim", "replace:_:-"], + "description": "Field with underscore to hyphen replacement" + } + }, + "primary_key": ["field1", "field2"], + "constraints": [ + { + "field": "field1", + "regex": "^[a-z0-9][a-z0-9._-]{0,127}$", + "description": "Must start with alphanumeric, can contain dots, underscores, hyphens" + }, + { + "field": "field2", + "regex": "^[a-z0-9][a-z0-9._-]{0,127}$", + "when_present": true, + "description": "Same pattern but only validated when present" + } + ] +} diff --git a/templates/pipeline_template.json b/templates/pipeline_template.json new file mode 100644 index 0000000..19ccd63 --- /dev/null +++ b/templates/pipeline_template.json @@ -0,0 +1,170 @@ +{ + "description": "Template for common pipeline patterns", + "patterns": { + "simple_read": { + "description": "Simple read operation with authentication and caching", + "pipeline": [ + { + "op": "auth.require_scopes", + "args": {"scopes": ["read"]} + }, + { + "op": "parse.path", + "args": {"entity": "artifact"} + }, + { + "op": "cache.get", + "args": { + "kind": "response", + "key": "cache_key/{namespace}/{name}", + "hit_out": "cache_hit", + "value_out": "cached_data" + } + }, + { + "op": "respond.json", + "args": { + "when": {"equals": ["$cache_hit", true]}, + "status": 200, + "body": "$cached_data" + } + }, + { + "op": "kv.get", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}", + "out": "data" + } + }, + { + "op": "cache.put", + "args": { + "kind": "response", + "key": "cache_key/{namespace}/{name}", + "ttl_seconds": 300, + "value": "$data" + } + }, + { + "op": "respond.json", + "args": { + "status": 200, + "body": "$data" + } + } + ] + }, + "transactional_write": { + "description": "Write operation with transaction and event emission", + "pipeline": [ + { + "op": "auth.require_scopes", + "args": {"scopes": ["write"]} + }, + { + "op": "parse.path", + "args": {"entity": "artifact"} + }, + { + "op": "parse.json", + "args": {"out": "body"} + }, + { + "op": "txn.begin", + "args": {"isolation": "serializable"} + }, + { + "op": "time.now_iso8601", + "args": {"out": "timestamp"} + }, + { + "op": "kv.cas_put", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}", + "if_absent": true, + "value": { + "namespace": "{namespace}", + "name": "{name}", + "data": "$body", + "created_at": "$timestamp" + } + } + }, + { + "op": "emit.event", + "args": { + "type": "resource.created", + "payload": { + "namespace": "{namespace}", + "name": "{name}", + "at": "$timestamp" + } + } + }, + { + "op": "txn.commit", + "args": {} + }, + { + "op": "respond.json", + "args": { + "status": 201, + "body": {"ok": true} + } + } + ] + }, + "proxy_with_fallback": { + "description": "Try local, fallback to upstream proxy", + "pipeline": [ + { + "op": "auth.require_scopes", + "args": {"scopes": ["read"]} + }, + { + "op": "kv.get", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}", + "out": "local_data" + } + }, + { + "op": "respond.json", + "args": { + "when": {"is_not_null": "$local_data"}, + "status": 200, + "body": "$local_data" + } + }, + { + "op": "proxy.fetch", + "args": { + "upstream": "originA", + "method": "GET", + "path": "/v1/{namespace}/{name}", + "out": "upstream_resp" + } + }, + { + "op": "respond.error", + "args": { + "when": {"not_in": ["$upstream_resp.status", [200]]}, + "status": 502, + "code": "UPSTREAM_ERROR", + "message": "Upstream fetch failed" + } + }, + { + "op": "respond.json", + "args": { + "status": 200, + "body": "$upstream_resp.body" + } + } + ] + } + } +} diff --git a/templates/route_template.json b/templates/route_template.json new file mode 100644 index 0000000..52744e2 --- /dev/null +++ b/templates/route_template.json @@ -0,0 +1,69 @@ +{ + "id": "my_custom_route", + "method": "GET", + "path": "/v1/{namespace}/{name}/custom-endpoint", + "tags": ["public"], + "description": "Template for a custom API route with full pipeline", + "pipeline": [ + { + "op": "auth.require_scopes", + "args": { + "scopes": ["read"] + }, + "comment": "Require read scope for access" + }, + { + "op": "parse.path", + "args": { + "entity": "artifact" + }, + "comment": "Parse path parameters into entity" + }, + { + "op": "normalize.entity", + "args": { + "entity": "artifact" + }, + "comment": "Normalize entity fields" + }, + { + "op": "validate.entity", + "args": { + "entity": "artifact" + }, + "comment": "Validate entity against constraints" + }, + { + "op": "kv.get", + "args": { + "doc": "artifact_meta", + "key": "artifact/{namespace}/{name}/metadata", + "out": "metadata" + }, + "comment": "Fetch metadata from KV store" + }, + { + "op": "respond.error", + "args": { + "when": { + "is_null": "$metadata" + }, + "status": 404, + "code": "NOT_FOUND", + "message": "Resource not found" + }, + "comment": "Return 404 if metadata doesn't exist" + }, + { + "op": "respond.json", + "args": { + "status": 200, + "body": { + "ok": true, + "data": "$metadata" + } + }, + "comment": "Return success response with metadata" + } + ] +} diff --git a/templates/upstream_template.json b/templates/upstream_template.json new file mode 100644 index 0000000..65d95d7 --- /dev/null +++ b/templates/upstream_template.json @@ -0,0 +1,36 @@ +{ + "name": "my_upstream", + "description": "Configuration for upstream repository proxy", + "base_url": "https://registry.example.com", + "auth": { + "mode": "bearer_token", + "description": "Authentication mode: none, basic, bearer_token, api_key", + "token_env": "UPSTREAM_TOKEN", + "alternatives": { + "basic": { + "username": "user", + "password_env": "UPSTREAM_PASSWORD" + }, + "api_key": { + "header": "X-API-Key", + "key_env": "UPSTREAM_API_KEY" + } + } + }, + "timeouts_ms": { + "connect": 2000, + "read": 10000, + "description": "Connection timeout: 2s, Read timeout: 10s" + }, + "retry": { + "max_attempts": 2, + "backoff_ms": 200, + "exponential": true, + "description": "Retry up to 2 times with exponential backoff" + }, + "health_check": { + "enabled": true, + "endpoint": "/health", + "interval_seconds": 60 + } +}