diff --git a/.github/workflows/todo-to-issues.yml b/.github/workflows/todo-to-issues.yml new file mode 100644 index 000000000..e67d9b72a --- /dev/null +++ b/.github/workflows/todo-to-issues.yml @@ -0,0 +1,162 @@ +name: TODO to Issues Sync + +# This workflow can be triggered manually to convert TODO items to GitHub issues +# or can be run on a schedule to keep issues in sync with TODO files + +on: + workflow_dispatch: + inputs: + mode: + description: 'Execution mode' + required: true + type: choice + options: + - dry-run + - export-json + - create-issues + default: 'dry-run' + + filter_priority: + description: 'Filter by priority (leave empty for all)' + required: false + type: choice + options: + - '' + - critical + - high + - medium + - low + + filter_label: + description: 'Filter by label (e.g., security, frontend)' + required: false + type: string + + exclude_checklist: + description: 'Exclude checklist items' + required: false + type: boolean + default: true + + limit: + description: 'Limit number of issues (0 for no limit)' + required: false + type: number + default: 0 + + # Uncomment to run on a schedule (e.g., weekly) + # schedule: + # - cron: '0 0 * * 0' # Every Sunday at midnight + +jobs: + convert-todos: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install GitHub CLI + run: | + type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y) + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y + + - name: Authenticate GitHub CLI + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "$GH_TOKEN" | gh auth login --with-token + gh auth status + + - name: Build command arguments + id: args + run: | + ARGS="" + + # Add mode + if [ "${{ inputs.mode }}" = "dry-run" ]; then + ARGS="$ARGS --dry-run" + elif [ "${{ inputs.mode }}" = "export-json" ]; then + ARGS="$ARGS --output todos-export.json" + elif [ "${{ inputs.mode }}" = "create-issues" ]; then + ARGS="$ARGS --create" + fi + + # Add filters + if [ -n "${{ inputs.filter_priority }}" ]; then + ARGS="$ARGS --filter-priority ${{ inputs.filter_priority }}" + fi + + if [ -n "${{ inputs.filter_label }}" ]; then + ARGS="$ARGS --filter-label ${{ inputs.filter_label }}" + fi + + if [ "${{ inputs.exclude_checklist }}" = "true" ]; then + ARGS="$ARGS --exclude-checklist" + fi + + # Add limit if specified + if [ "${{ inputs.limit }}" != "0" ]; then + ARGS="$ARGS --limit ${{ inputs.limit }}" + fi + + echo "args=$ARGS" >> $GITHUB_OUTPUT + echo "Command arguments: $ARGS" + + - name: Run populate-kanban script + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 tools/project-management/populate-kanban.py ${{ steps.args.outputs.args }} + + - name: Upload JSON export (if applicable) + if: inputs.mode == 'export-json' + uses: actions/upload-artifact@v4 + with: + name: todos-export + path: todos-export.json + retention-days: 30 + + - name: Create summary + if: always() + run: | + echo "## TODO to Issues Conversion" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Mode:** ${{ inputs.mode }}" >> $GITHUB_STEP_SUMMARY + + if [ -n "${{ inputs.filter_priority }}" ]; then + echo "**Priority Filter:** ${{ inputs.filter_priority }}" >> $GITHUB_STEP_SUMMARY + fi + + if [ -n "${{ inputs.filter_label }}" ]; then + echo "**Label Filter:** ${{ inputs.filter_label }}" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ inputs.exclude_checklist }}" = "true" ]; then + echo "**Checklist Items:** Excluded" >> $GITHUB_STEP_SUMMARY + fi + + if [ "${{ inputs.limit }}" != "0" ]; then + echo "**Limit:** ${{ inputs.limit }} items" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ inputs.mode }}" = "export-json" ]; then + echo "āœ… JSON export created successfully" >> $GITHUB_STEP_SUMMARY + echo "Download the artifact from the workflow run page" >> $GITHUB_STEP_SUMMARY + elif [ "${{ inputs.mode }}" = "create-issues" ]; then + echo "āœ… GitHub issues created successfully" >> $GITHUB_STEP_SUMMARY + echo "View issues: https://github.com/${{ github.repository }}/issues" >> $GITHUB_STEP_SUMMARY + else + echo "ā„¹ļø Dry run completed - no issues created" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.gitignore b/.gitignore index 1b1636563..9bbaa6a53 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,11 @@ lint-output.txt stub-patterns.json complexity-report.json +# TODO management +todos-baseline.json +todos-export.json +todos*.json + # Project-specific **/agent-eval-report* vite.config.ts.bak* diff --git a/package.json b/package.json index c2e835cec..e87301e09 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "todos:export-filtered": "python3 tools/project-management/populate-kanban.py --output todos-filtered.json --exclude-checklist", "todos:test": "python3 tools/project-management/test_populate_kanban.py", "todos:create": "python3 tools/project-management/populate-kanban.py --create", - "todos:help": "python3 tools/project-management/populate-kanban.py --help" + "todos:help": "python3 tools/project-management/populate-kanban.py --help", + "todos:check": "python3 tools/project-management/check-new-todos.py", + "todos:baseline": "python3 tools/project-management/check-new-todos.py --save-baseline" }, "devDependencies": { "prisma": "^7.2.0" diff --git a/tools/project-management/README.md b/tools/project-management/README.md index 7e89e5305..41b4d6d8c 100644 --- a/tools/project-management/README.md +++ b/tools/project-management/README.md @@ -2,6 +2,35 @@ Tools for managing MetaBuilder's GitHub project board and issues. +## Overview + +This directory contains three main tools: + +1. **populate-kanban.py** - Convert TODO items to GitHub issues +2. **check-new-todos.py** - Monitor for new TODO items +3. **test_populate_kanban.py** - Unit tests for the conversion script + +## Quick Start + +```bash +# From repository root: + +# Preview issues that would be created +npm run todos:preview + +# Run tests +npm run todos:test + +# Check for new TODOs since baseline +npm run todos:check + +# Export to JSON +npm run todos:export + +# Create issues on GitHub (requires gh auth) +npm run todos:create +``` + ## populate-kanban.py Automatically populate the GitHub project kanban board from TODO markdown files. @@ -310,8 +339,152 @@ From the last full parse: - **Medium priority**: 269 items - **Low priority**: 80 items -### See Also +--- -- [docs/todo/README.md](../../docs/todo/README.md) - TODO system overview -- [docs/todo/TODO_STATUS.md](../../docs/todo/TODO_STATUS.md) - Current status -- [GitHub Projects](https://github.com/users/johndoe6345789/projects/2) - Target kanban board +## check-new-todos.py + +Monitor for new TODO items added since the last baseline. + +### Features + +- **Baseline tracking**: Save current TODO state as baseline +- **Change detection**: Identify new and removed TODO items +- **Diff reporting**: Show what changed and where +- **CI integration**: Exit code indicates if new items found + +### Quick Start + +```bash +# Save current state as baseline +npm run todos:baseline + +# Check for new items +npm run todos:check + +# Or use directly: +python3 tools/project-management/check-new-todos.py --save-baseline +python3 tools/project-management/check-new-todos.py +``` + +### Use Cases + +1. **CI/CD Integration**: Fail the build if new TODOs are added +2. **PR Reviews**: Detect if a PR adds new TODOs +3. **Project Management**: Track TODO growth over time +4. **Issue Creation**: Know exactly which items are new + +### Example Output + +``` +============================================================ +TODO Items Comparison +============================================================ +Baseline count: 775 +Current count: 783 +Net change: +8 + +New items: 8 +Removed items: 0 + +============================================================ +New TODO Items (8) +============================================================ + +1. Add GraphQL API support + File: docs/todo/features/15-API-TODO.md:23 + Section: API Features + Priority: šŸŸ  High + Labels: feature, backend + +2. Implement caching layer + File: docs/todo/infrastructure/16-PERFORMANCE-TODO.md:45 + Section: Optimization + Priority: šŸŸ” Medium + Labels: infrastructure, performance + +... + +============================================================ +šŸ’” Tip: Create issues for these new items: + python3 tools/project-management/populate-kanban.py --create --limit 8 +============================================================ +``` + +### CI/CD Example + +```yaml +# .github/workflows/check-todos.yml +name: Check for new TODOs + +on: + pull_request: + paths: + - 'docs/todo/**' + +jobs: + check-todos: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check for new TODOs + run: | + python3 tools/project-management/check-new-todos.py + if [ $? -eq 1 ]; then + echo "::warning::New TODO items detected. Consider creating issues." + fi +``` + +--- + +## test_populate_kanban.py + +Comprehensive unit tests for populate-kanban.py. + +### Features + +- **15 test cases** covering all major functionality +- **Parser tests**: TODO extraction, context, sections +- **Categorization tests**: Labels, priorities, filters +- **Edge case handling**: Empty items, long titles, special files + +### Running Tests + +```bash +# Run all tests +npm run todos:test + +# Or directly +python3 tools/project-management/test_populate_kanban.py + +# With pytest (if installed) +pytest tools/project-management/test_populate_kanban.py -v +``` + +### Test Coverage + +- āœ… Parse simple TODO items +- āœ… Skip empty/short items +- āœ… Extract context from surrounding lines +- āœ… Track section headers +- āœ… Track line numbers +- āœ… Categorize by filename +- āœ… Categorize by directory +- āœ… Assign priorities from README +- āœ… Assign default priorities +- āœ… Exclude special files (README, STATUS, SCAN) +- āœ… Truncate long titles +- āœ… Dry run mode +- āœ… Issue creation +- āœ… Project board integration +- āœ… Filter logic + +--- + +## See Also + +- **[docs/guides/TODO_TO_ISSUES.md](../../docs/guides/TODO_TO_ISSUES.md)** - Complete user guide +- **[docs/todo/README.md](../../docs/todo/README.md)** - TODO system overview +- **[docs/todo/TODO_STATUS.md](../../docs/todo/TODO_STATUS.md)** - Current status +- **[KANBAN_READY.md](../../KANBAN_READY.md)** - Implementation summary +- **[GitHub Projects](https://github.com/users/johndoe6345789/projects/2)** - Target kanban board +- **[.github/workflows/todo-to-issues.yml](../../.github/workflows/todo-to-issues.yml)** - GitHub Action workflow diff --git a/tools/project-management/check-new-todos.py b/tools/project-management/check-new-todos.py new file mode 100755 index 000000000..2f7f11a86 --- /dev/null +++ b/tools/project-management/check-new-todos.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Check for new TODO items added since last check. + +This script compares the current TODO items with a baseline and reports new items. +Useful for detecting when new TODOs are added that should become GitHub issues. + +Usage: + python3 check-new-todos.py [--baseline FILE] [--save-baseline] +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Set, List + +# Import TodoParser from populate-kanban.py +import importlib.util +script_dir = Path(__file__).parent +spec = importlib.util.spec_from_file_location("populate_kanban", script_dir / "populate-kanban.py") +populate_kanban = importlib.util.module_from_spec(spec) +spec.loader.exec_module(populate_kanban) +TodoParser = populate_kanban.TodoParser + + +def get_todo_signatures(items: List) -> Set[str]: + """Generate unique signatures for TODO items.""" + signatures = set() + for item in items: + # Create signature from file, line number, and truncated title + sig = f"{item.file}:{item.line_number}:{item.title[:50]}" + signatures.add(sig) + return signatures + + +def main(): + parser = argparse.ArgumentParser( + description='Check for new TODO items', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__ + ) + + parser.add_argument( + '--baseline', + type=Path, + default=Path('todos-baseline.json'), + help='Path to baseline file (default: todos-baseline.json)' + ) + + parser.add_argument( + '--save-baseline', + action='store_true', + help='Save current TODOs as new baseline' + ) + + parser.add_argument( + '--todo-dir', + type=Path, + help='Path to docs/todo directory (default: auto-detect)' + ) + + args = parser.parse_args() + + # Auto-detect todo directory if not specified + if args.todo_dir is None: + repo_root = script_dir.parent.parent + args.todo_dir = repo_root / 'docs' / 'todo' + + if not args.todo_dir.exists(): + print(f"ERROR: TODO directory not found: {args.todo_dir}") + sys.exit(1) + + # Parse current TODOs + print(f"Parsing TODO files from: {args.todo_dir}") + parser_obj = TodoParser(args.todo_dir) + current_items = parser_obj.parse_all() + current_signatures = get_todo_signatures(current_items) + + print(f"Found {len(current_items)} TODO items") + + # Save baseline if requested + if args.save_baseline: + baseline_data = { + 'count': len(current_items), + 'signatures': list(current_signatures) + } + with open(args.baseline, 'w', encoding='utf-8') as f: + json.dump(baseline_data, f, indent=2) + print(f"\nāœ“ Saved baseline to: {args.baseline}") + return + + # Load baseline for comparison + if not args.baseline.exists(): + print(f"\nā„¹ļø No baseline file found at: {args.baseline}") + print("Run with --save-baseline to create initial baseline") + sys.exit(0) + + with open(args.baseline, 'r', encoding='utf-8') as f: + baseline_data = json.load(f) + + baseline_signatures = set(baseline_data['signatures']) + baseline_count = baseline_data['count'] + + # Find new items + new_signatures = current_signatures - baseline_signatures + removed_signatures = baseline_signatures - current_signatures + + print(f"\n{'='*60}") + print(f"TODO Items Comparison") + print(f"{'='*60}") + print(f"Baseline count: {baseline_count}") + print(f"Current count: {len(current_items)}") + print(f"Net change: {len(current_items) - baseline_count:+d}") + print(f"\nNew items: {len(new_signatures)}") + print(f"Removed items: {len(removed_signatures)}") + + # Show new items + if new_signatures: + print(f"\n{'='*60}") + print(f"New TODO Items ({len(new_signatures)})") + print(f"{'='*60}") + + # Find full details of new items + new_items = [ + item for item in current_items + if f"{item.file}:{item.line_number}:{item.title[:50]}" in new_signatures + ] + + for i, item in enumerate(new_items[:20], 1): # Show first 20 + print(f"\n{i}. {item.title[:80]}") + print(f" File: {item.file}:{item.line_number}") + print(f" Section: {item.section}") + print(f" Priority: {item.priority}") + print(f" Labels: {', '.join(item.labels)}") + + if len(new_items) > 20: + print(f"\n... and {len(new_items) - 20} more") + + print(f"\n{'='*60}") + print(f"šŸ’” Tip: Create issues for these new items:") + print(f" python3 tools/project-management/populate-kanban.py --create --limit {len(new_items)}") + print(f"{'='*60}") + + # Show removed items + if removed_signatures: + print(f"\n{'='*60}") + print(f"Removed TODO Items ({len(removed_signatures)})") + print(f"{'='*60}") + + for i, sig in enumerate(list(removed_signatures)[:10], 1): + print(f"{i}. {sig}") + + if len(removed_signatures) > 10: + print(f"... and {len(removed_signatures) - 10} more") + + # Exit with status code based on changes + if new_signatures: + sys.exit(1) # New items found + else: + print("\nāœ“ No new TODO items since baseline") + sys.exit(0) + + +if __name__ == '__main__': + main()