From c1856401760a6fca5dec668bc0ee4508b450d4a0 Mon Sep 17 00:00:00 2001 From: Shawn Anderson Date: Thu, 9 Oct 2025 18:27:20 -0700 Subject: [PATCH] Add SDG network visualization system and write command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SDG Network System: - SDG_NETWORK_ANALYSIS.md: Analysis of SDG interconnections - SDG_NETWORK_GUIDE.md: User guide for SDG network visualizations - specs/sdg_network_progressive.md: Progressive learning specification - specs/sdg_network_url_strategy.json: Curated D3 network visualization URLs - sdg_viz/: 3 generated SDG network visualizations Commands: - .claude/commands/write.md: Custom write command for Claude Code Extends web-enhanced infinite loop to sustainability data visualization. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .claude/commands/write.md | 1 + SDG_NETWORK_ANALYSIS.md | 540 ++++++++++++++ SDG_NETWORK_GUIDE.md | 404 +++++++++++ sdg_viz/sdg_viz_1.html | 431 ++++++++++++ sdg_viz/sdg_viz_2.html | 665 ++++++++++++++++++ sdg_viz/sdg_viz_3.html | 1014 +++++++++++++++++++++++++++ sdg_viz/sdg_viz_4.html | 720 +++++++++++++++++++ sdg_viz/sdg_viz_5.html | 739 +++++++++++++++++++ sdg_viz/sdg_viz_6.html | 974 +++++++++++++++++++++++++ sdg_viz/sdg_viz_7.html | 904 ++++++++++++++++++++++++ sdg_viz/sdg_viz_8.html | 970 +++++++++++++++++++++++++ sdg_viz/sdg_viz_9.html | 916 ++++++++++++++++++++++++ specs/sdg_network_progressive.md | 446 ++++++++++++ specs/sdg_network_url_strategy.json | 278 ++++++++ 14 files changed, 9002 insertions(+) create mode 100644 .claude/commands/write.md create mode 100644 SDG_NETWORK_ANALYSIS.md create mode 100644 SDG_NETWORK_GUIDE.md create mode 100644 sdg_viz/sdg_viz_1.html create mode 100644 sdg_viz/sdg_viz_2.html create mode 100644 sdg_viz/sdg_viz_3.html create mode 100644 sdg_viz/sdg_viz_4.html create mode 100644 sdg_viz/sdg_viz_5.html create mode 100644 sdg_viz/sdg_viz_6.html create mode 100644 sdg_viz/sdg_viz_7.html create mode 100644 sdg_viz/sdg_viz_8.html create mode 100644 sdg_viz/sdg_viz_9.html create mode 100644 specs/sdg_network_progressive.md create mode 100644 specs/sdg_network_url_strategy.json diff --git a/.claude/commands/write.md b/.claude/commands/write.md new file mode 100644 index 0000000..0ea381d --- /dev/null +++ b/.claude/commands/write.md @@ -0,0 +1 @@ +OK can you create a manual in @ai_docs/ that explains everything that we accomplished today? diff --git a/SDG_NETWORK_ANALYSIS.md b/SDG_NETWORK_ANALYSIS.md new file mode 100644 index 0000000..8f7c4d4 --- /dev/null +++ b/SDG_NETWORK_ANALYSIS.md @@ -0,0 +1,540 @@ +# SDG Network Visualization - Analysis & Strategy + +## Executive Summary + +I've analyzed the existing SDG network modelling project and designed a **web-enhanced infinite loop system** that will progressively generate sophisticated force-directed network visualizations with automatic API discovery. + +### What Was Reviewed + +**Existing SDG Network Project** (`ai_docs/sdg-network-modelling/`): +- Python notebooks fetching UN SDG API data +- D3.js v3 force-directed graph visualization +- Basic network showing: Geographic areas ↔ Ecosystem Capitals ↔ SDGs ↔ Targets +- Simple interactions: hover tooltips, drag nodes +- Single API source (UN Statistics) + +**Current Limitations:** +- Outdated D3.js version (v3 vs current v7) +- Single hardcoded API +- Basic visual design (simple circles/lines) +- Limited interactivity (just hover) +- No progressive enhancement +- No web learning integration + +## New System Design + +### Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Web-Enhanced Infinite Loop (/project:infinite-web) │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Phase 0: │ → │ Phase 1: │ → │ Phase 2: │ │ +│ │ Web Priming │ │ Spec + Web │ │ API │ │ +│ │ │ │ Analysis │ │ Discovery │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ ↓ ↓ ↓ │ +│ Fetch D3.js docs Read spec file Search for APIs │ +│ Learn force graphs + priming knowledge Parse Swagger │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Phase 3: │ → │ Phase 4: │ → │ Phase 5: │ │ +│ │ Iteration │ │ Parallel │ │ Quality │ │ +│ │ Strategy │ │ Agents │ │ Assurance │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ ↓ ↓ ↓ │ +│ Map URLs to iters Launch sub-agents Verify outputs │ +│ Plan API sources Each learns + builds Check API usage │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Progressive Enhancement Strategy + +**Phase 1: Foundation (Iterations 1-5)** +- Modern D3.js v7 +- Single API per iteration +- Node sizing, color coding, labels +- Basic interactivity (drag, zoom, click) +- Learn from: D3.js fundamentals, force simulation basics + +**Phase 2: Intermediate (Iterations 6-12)** +- Combine 2-3 APIs +- Multi-property encoding (color + border) +- Edge bundling, curved edges +- Search, filter, side panels +- Learn from: Advanced D3 techniques, data integration + +**Phase 3: Advanced (Iterations 13-20)** +- 4+ API sources +- Clustering algorithms, community detection +- Real-time data updates +- Graph metrics dashboards +- Learn from: Graph algorithms, streaming data + +**Phase 4: Expert (Iterations 21+)** +- Federated API queries +- ML-based layouts +- AI-powered insights +- 3D/VR capabilities +- Learn from: Cutting-edge research, novel techniques + +## API Discovery System + +### Automatic Discovery Process + +Each iteration performs web searches to find new APIs: + +```javascript +// Search patterns (from sdg_network_url_strategy.json) +"environmental data API swagger site:github.com" +"scientific data REST API openapi" +"sustainable development goals API JSON" +"atmospheric weather data API free" +"water quality watershed API REST" +"forest biodiversity data API open source" +"NOAA ocean marine data API" +"NASA earth observation API" +``` + +### Target API Domains + +**Environmental:** +- OpenAQ (air quality) +- EPA APIs (environmental protection) +- USGS Water Services +- Global Forest Watch + +**Scientific:** +- GBIF (biodiversity) +- NASA Earthdata +- NOAA (ocean, weather, climate) +- Seismic/earthquake data (USGS) + +**Sustainable Development:** +- UN SDG API (baseline) +- World Bank Open Data +- OECD Statistics +- WHO Global Health +- FAO Agricultural Data + +**Data Integration:** +- Swagger/OpenAPI spec parsing +- Automatic entity mapping to nodes/edges +- Cross-API entity resolution +- Data quality assessment + +## Network Data Model + +### Node Structure +```javascript +{ + id: "unique_id", + name: "Display Name", + type: "entity_type", // country, indicator, species, etc. + group: "category", // for color coding + size: value, // visual size from data + properties: { + value: number, + unit: "string", + timestamp: "date", + source_api: "api_name", + // ... domain-specific attributes + } +} +``` + +### Edge Structure +```javascript +{ + source: "node_id", + target: "node_id", + type: "relationship", // correlation, dependency, etc. + strength: value, // for force simulation + weight: value, // visual width + properties: { + correlation: number, + confidence: number, + source_api: "api_name" + } +} +``` + +## Visual Enhancement Progression + +### Node Enhancements +**Foundation:** +- Size ← data magnitude +- Color ← category +- Labels with smart positioning + +**Intermediate:** +- Border color ← secondary property +- Icon overlays ← entity type +- Clustering ← community detection + +**Advanced:** +- Shape ← entity class +- Opacity ← confidence level +- Animation ← state changes + +**Expert:** +- 3D projection ← multi-dimensional data +- ML-based positioning +- Anomaly highlighting + +### Edge Enhancements +**Foundation:** +- Width ← relationship strength +- Color ← relationship type +- Arrows ← directionality + +**Intermediate:** +- Curved paths ← visibility +- Bundling ← clarity +- Dash patterns ← relationship class + +**Advanced:** +- Animated flow ← data transfer +- Dynamic strength ← real-time values +- Hierarchical bundling ← complex relationships + +**Expert:** +- Probabilistic rendering +- Predicted relationships +- Causal inference visualization + +## Web Learning Integration + +### URL Strategy Structure + +The system uses `specs/sdg_network_url_strategy.json` with: + +**Foundation URLs (Iterations 1-5):** +- D3.js fundamentals +- Basic force-directed graphs +- Color scales and node styling +- Interactive tooltips + +**Intermediate URLs (Iterations 6-12):** +- Force-based clustering +- Custom force implementations +- Advanced color schemes +- Animated transitions + +**Advanced URLs (Iterations 13-20):** +- Edge bundling algorithms +- 3D projections +- Adaptive rendering +- Real-time data integration + +**Expert URLs (Iterations 21+):** +- Graph theory algorithms +- ML-enhanced layouts +- Multi-view interfaces +- Export capabilities + +### Learning Process Per Iteration + +1. **WebFetch** assigned URL +2. **Extract** 1-3 specific techniques +3. **Implement** in visualization +4. **Document** source and learnings +5. **Demonstrate** improvement + +## File Structure + +### Created Files + +``` +specs/ +├── sdg_network_progressive.md # Main specification +└── sdg_network_url_strategy.json # Learning URLs + +SDG_NETWORK_GUIDE.md # Complete usage guide +SDG_NETWORK_ANALYSIS.md # This file (analysis) +CLAUDE.md # Updated with SDG commands +``` + +### Output Structure + +``` +sdg_viz/ # Created by command +├── sdg_viz_1.html # Iteration 1 +├── sdg_viz_2.html # Iteration 2 +├── ... +└── sdg_viz_N.html # Iteration N +``` + +Each HTML file is self-contained with: +- D3.js v7 force-directed graph +- API data fetching +- Interactive controls +- Documentation footer (APIs used, web learning) + +## Usage Examples + +### Quick Start +```bash +claude +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 +``` + +Output: `sdg_viz/sdg_viz_1.html` +- UN SDG API baseline +- Basic D3.js force graph +- Foundation-level enhancements + +### Small Batch +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 +``` + +Output: 5 files, each with: +- Different API source +- Progressive D3 techniques +- Building on previous iterations + +### Full Progressive Learning +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 12 specs/sdg_network_url_strategy.json +``` + +Output: 12 iterations +- Iterations 1-5: Foundation (basic graphs, single APIs) +- Iterations 6-12: Intermediate (multi-API, advanced interactivity) + +### Infinite Mode +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite specs/sdg_network_url_strategy.json +``` + +Runs until context limit: +- Progressive difficulty waves +- Automatic API discovery +- Increasingly sophisticated visualizations + +## Key Innovations + +### 1. Automatic API Discovery +- **Old**: Hardcoded UN SDG API only +- **New**: Web searches find Swagger/OpenAPI endpoints +- **Impact**: Unlimited data source expansion + +### 2. Progressive Web Learning +- **Old**: Static implementation +- **New**: Each iteration learns from curated web resources +- **Impact**: Continuous technique improvement + +### 3. Multi-Source Integration +- **Old**: Single API +- **New**: Combine 2-5+ APIs per iteration +- **Impact**: Richer, more insightful networks + +### 4. Modern D3.js +- **Old**: D3.js v3 (2012) +- **New**: D3.js v7 (2023) +- **Impact**: Better performance, modern features + +### 5. Rich Interactivity +- **Old**: Basic hover tooltip +- **New**: Search, filter, zoom, click actions, exports, real-time updates +- **Impact**: Deeper data exploration + +## Quality Metrics + +Each iteration evaluated on: + +### Data Richness +- API sources count (1 → 5+) +- Node count +- Edge count +- Update frequency + +### Visual Quality +- Color sophistication +- Label readability +- Animation smoothness (60fps target) +- Layout clarity score + +### Interactivity +- Feature count +- Response time +- Filter capabilities +- Information density + +### Technical Innovation +- New techniques applied +- Performance metrics +- Code quality +- Browser compatibility + +## Comparison: Old vs New + +| Aspect | Old System | New System | +|--------|-----------|------------| +| **D3 Version** | v3 (2012) | v7 (2023) | +| **APIs** | 1 (hardcoded) | Unlimited (auto-discovery) | +| **Node Styling** | Basic circles | Size, color, border, icons, shapes | +| **Edge Styling** | Simple lines | Width, color, curves, bundling, animation | +| **Interactivity** | Hover only | Hover, click, drag, zoom, search, filter | +| **Learning** | None | Web-sourced techniques per iteration | +| **Data Sources** | UN SDG only | Environmental, scientific, SDG, health, etc. | +| **Architecture** | Static | Progressive enhancement | +| **Output** | Single file | Iteration series | + +## Next Steps + +### Immediate Actions + +1. **Test Single Iteration:** + ```bash + claude + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 + ``` + +2. **Review Output:** + - Open `sdg_viz/sdg_viz_1.html` in browser + - Check UN SDG API integration + - Test basic interactivity + - Verify web learning attribution + +3. **Generate Small Batch:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 + ``` + +4. **Compare Iterations:** + - See API diversity + - Note progressive improvements + - Identify new techniques applied + +### Advanced Exploration + +1. **Custom API Focus:** + - Modify spec to target specific domains (ocean, climate, etc.) + - Adjust search templates for specific APIs + +2. **Algorithm Comparison:** + - Generate iterations with different force algorithms + - Compare layout effectiveness + +3. **Real-time Dashboards:** + - Generate iterations 13-20 for real-time features + - Test streaming data integration + +4. **Infinite Mode:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite specs/sdg_network_url_strategy.json + ``` + +### Customization Options + +**Create Domain-Specific Specs:** +```bash +# Ocean/marine networks +cp specs/sdg_network_progressive.md specs/ocean_network.md +# Edit to focus on ocean APIs + +# Climate change networks +cp specs/sdg_network_progressive.md specs/climate_network.md +# Edit to focus on climate APIs +``` + +**Extend URL Strategy:** +- Add new D3 technique URLs +- Add domain-specific learning resources +- Create custom search templates + +**Modify Output Format:** +- Change from single HTML to multi-file +- Add server-side API proxies +- Generate dashboards with multiple views + +## Success Indicators + +### Technical Success +- ✅ Modern D3.js v7 implementation +- ✅ Multiple API sources per iteration +- ✅ Self-contained HTML files +- ✅ 60fps animations +- ✅ No console errors + +### Functional Success +- ✅ Force-directed graphs render correctly +- ✅ API data fetches successfully +- ✅ Interactivity works smoothly +- ✅ Visualizations are informative +- ✅ Error handling is graceful + +### Learning Success +- ✅ Each iteration demonstrates new technique +- ✅ Web sources are properly attributed +- ✅ Progressive improvement is evident +- ✅ API diversity increases +- ✅ Complexity grows appropriately + +### User Value +- ✅ Insights are actionable +- ✅ Exploration is intuitive +- ✅ Data relationships are clear +- ✅ SDG analysis is enhanced +- ✅ Visualizations are shareable + +## Troubleshooting Guide + +### API Access Issues +**Problem**: API requires authentication +**Solution**: Iterations handle gracefully with fallback APIs + +**Problem**: CORS errors +**Solution**: Use CORS proxies or alternative APIs + +**Problem**: Rate limiting +**Solution**: Implement caching and request throttling + +### Performance Issues +**Problem**: Large graphs lag +**Solution**: Canvas rendering (implemented in advanced iterations) + +**Problem**: Slow force simulation +**Solution**: Adaptive sampling, LOD techniques + +### Browser Compatibility +**Problem**: Old browser issues +**Solution**: Target modern browsers, provide fallbacks + +## Resources + +### Documentation +- `SDG_NETWORK_GUIDE.md` - Complete usage guide +- `specs/sdg_network_progressive.md` - Specification +- `specs/sdg_network_url_strategy.json` - Learning URLs +- `CLAUDE.md` - Project overview + +### Reference +- `ai_docs/sdg-network-modelling/` - Original baseline +- Observable HQ - D3.js examples +- GitHub public-apis - API directory + +### Commands +- `.claude/commands/infinite-web.md` - Implementation +- `/project:infinite-web` - Run command + +## Conclusion + +The new SDG Network Visualization system transforms a static, single-API visualization into a **progressive, web-learning-enhanced, multi-API discovery system** that: + +1. **Discovers** new data sources automatically via web search +2. **Learns** advanced techniques from curated web resources +3. **Integrates** multiple APIs into rich network visualizations +4. **Enhances** progressively across iterations +5. **Documents** sources and learnings clearly + +This system leverages the web-enhanced infinite loop pattern to create increasingly sophisticated SDG network visualizations that provide genuine value for sustainable development insights and data exploration. + +**The system is ready to run. Start with:** +```bash +claude +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 +``` diff --git a/SDG_NETWORK_GUIDE.md b/SDG_NETWORK_GUIDE.md new file mode 100644 index 0000000..8368001 --- /dev/null +++ b/SDG_NETWORK_GUIDE.md @@ -0,0 +1,404 @@ +# SDG Network Visualization - Web-Enhanced Infinite Loop Guide + +## Overview + +This guide explains how to use the web-enhanced infinite agentic loop to generate progressively sophisticated SDG (Sustainable Development Goals) network visualizations that integrate multiple open data APIs. + +## What Gets Generated + +Each iteration produces a **self-contained HTML file** with: +- **Force-directed network graph** using D3.js +- **Real data** from open APIs (environmental, scientific, SDG-related) +- **Progressive enhancements** in visualization quality and interactivity +- **Web-sourced techniques** applied and documented +- **API discovery** through automated web searches + +## Current Baseline + +The existing SDG network visualization (`ai_docs/sdg-network-modelling/`) provides: +- Basic D3.js v3 force-directed graph +- UN SDG API integration +- Simple node/edge rendering +- Basic hover tooltips +- Geographic and SDG goal relationships + +## Progressive Enhancement Strategy + +### **Phase 1: Foundation (Iterations 1-5)** +**Visual Enhancements:** +- Node size based on data magnitude +- Color coding by categories (using D3 color scales) +- Smart label positioning +- Rich hover tooltips with formatted data +- Edge width based on relationship strength +- Directional arrows and edge patterns + +**Interactivity:** +- Drag nodes to reposition +- Zoom and pan controls +- Click to highlight connected nodes +- Basic filtering by node type +- Legend for color coding + +**Data:** +- Single API per iteration +- Examples: UN SDG API, World Bank, OpenAQ, USGS Water + +### **Phase 2: Intermediate (Iterations 6-12)** +**Visual Enhancements:** +- Multi-property color encoding (fill color + border color) +- Icon overlays on nodes +- Curved edges for better visibility +- Node clustering by community +- Edge bundling for cleaner layouts +- Custom node shapes for different entity types + +**Interactivity:** +- Search functionality to find nodes +- Multi-select for comparison +- Time-based animation +- Expandable/collapsible groups +- Side panel with statistics +- Export visualization and data + +**Data:** +- Combine 2-3 API sources +- Automatic data merging +- Examples: NASA + NOAA + EPA, WHO + World Bank + UN + +### **Phase 3: Advanced (Iterations 13-20)** +**Visual Enhancements:** +- Force-based clustering algorithms +- Community detection visualization +- Centrality measures displayed +- Node importance ranking +- Hierarchical edge bundling +- Animated state transitions + +**Interactivity:** +- Real-time data updates +- Graph layout algorithm switching +- Advanced filtering (boolean logic) +- Shortest path finding +- Neighborhood exploration +- Graph metric dashboards + +**Data:** +- 4+ API sources integrated +- Automated API discovery via Swagger search +- Cross-API entity resolution +- Streaming data integration + +### **Phase 4: Expert (Iterations 21+)** +**Visual Enhancements:** +- ML-based optimal node positioning +- Anomaly detection highlighting +- Predictive attributes visualization +- 3D network projections +- VR/AR capabilities + +**Interactivity:** +- Natural language graph queries +- AI-powered insights +- Automatic pattern detection +- Graph comparison tools +- Collaborative features + +**Data:** +- Federated queries across APIs +- Knowledge graph construction +- Semantic web integration +- Blockchain data sources + +## API Discovery Process + +Each iteration automatically discovers new data sources: + +### **Target Domains:** +1. **Environmental**: Air quality, water quality, climate +2. **Scientific**: Biodiversity, satellites, research datasets +3. **Atmospheric**: Weather, air monitoring +4. **Ecological**: Forests, species, conservation +5. **Ocean**: Marine life, coastal data +6. **Geophysical**: Earthquakes, volcanoes +7. **Health**: Disease, mortality, healthcare +8. **SDG**: Development indicators + +### **Discovery Method:** +1. Web search: `"[domain] API swagger site:github.com"` +2. Parse Swagger/OpenAPI specifications +3. Evaluate API quality and relevance +4. Map data to network structure (nodes/edges) +5. Integrate and visualize + +### **API Examples:** +- **UN SDG API**: https://unstats.un.org/SDGAPI/v1/sdg/ +- **OpenAQ**: Air quality data worldwide +- **USGS Water Services**: Water quality and flow +- **Global Forest Watch**: Deforestation and forest data +- **NASA Earthdata**: Satellite and earth science +- **NOAA**: Weather, ocean, climate data +- **GBIF**: Biodiversity occurrence data +- **World Bank**: Development indicators + +## Web Learning Integration + +Each iteration learns from curated web resources: + +### **Foundation Topics:** +- D3.js fundamentals and data binding +- Basic force-directed graphs +- Color scales and node styling +- Interactive tooltips and zoom +- Canvas rendering for performance + +### **Intermediate Topics:** +- Force-based clustering +- Custom force implementations +- Advanced color schemes +- Animated transitions +- Modular component architecture + +### **Advanced Topics:** +- Edge bundling algorithms +- 3D projections and rotations +- Adaptive rendering for large graphs +- Real-time streaming data +- Multi-view interfaces + +### **Expert Topics:** +- Graph theory algorithms (centrality, shortest path) +- Community detection +- Semantic zoom +- Export capabilities +- ML-enhanced layouts + +## Usage Examples + +### **Single Iteration with Default Settings:** +```bash +claude +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 +``` + +This generates `sdg_viz/sdg_viz_1.html` with: +- UN SDG API baseline +- Basic force graph from D3.js documentation +- Foundation-level enhancements + +### **Small Batch (5 iterations):** +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 +``` + +Generates 5 files with: +- Different API sources (UN SDG, World Bank, OpenAQ, etc.) +- Progressive techniques from foundation URLs +- Each builds upon insights from previous iterations + +### **Medium Batch with Custom URL Strategy:** +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 12 specs/sdg_network_url_strategy.json +``` + +Generates 12 iterations: +- Iterations 1-5: Foundation techniques +- Iterations 6-12: Intermediate techniques +- Multiple APIs combined +- Advanced interactivity + +### **Infinite Mode (Continuous until context limit):** +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite specs/sdg_network_url_strategy.json +``` + +Runs continuously with: +- Progressive difficulty waves +- Automatic API discovery via web search +- Increasingly sophisticated visualizations +- Stops gracefully at context limit + +## Expected Output Structure + +Each `sdg_viz_[N].html` file contains: + +```html + + + + SDG Network Visualization [N] - [Data Theme] + + + +
+ +
/* Interactive controls */
+
/* Data details */
+
/* Visual legend */
+
+ + + + +
+

Data Sources: [APIs used]

+

Web Learning: [URL + techniques]

+
+ + +``` + +## Key Improvements Over Baseline + +### **Data Integration:** +- ❌ **Old**: Single hardcoded API (UN SDG only) +- ✅ **New**: Automatic API discovery, multiple sources combined + +### **Visualization:** +- ❌ **Old**: Basic circles and lines, single color scheme +- ✅ **New**: Rich node/edge properties, dynamic sizing, multi-attribute encoding + +### **Interactivity:** +- ❌ **Old**: Simple hover tooltip +- ✅ **New**: Search, filter, zoom, click actions, side panels, exports + +### **Learning:** +- ❌ **Old**: Static implementation +- ✅ **New**: Each iteration learns from web and improves + +### **Technology:** +- ❌ **Old**: D3.js v3 (outdated) +- ✅ **New**: D3.js v7 (modern), latest techniques + +## Quality Metrics + +Each iteration is evaluated on: + +### **Data Richness:** +- Number of API sources (1 → 5+) +- Total nodes/edges +- Data update frequency + +### **Visual Quality:** +- Color coding sophistication +- Label readability +- Animation smoothness (60fps) +- Layout clarity + +### **Interactivity:** +- Number of features (hover → search → filter → AI) +- Response time +- Information density + +### **Technical Innovation:** +- New techniques applied +- Performance improvements +- Code quality + +## Advanced Use Cases + +### **Compare Algorithms:** +Generate iterations with different force algorithms: +```bash +# Each iteration learns a different layout algorithm +> /project:infinite-web specs/sdg_network_progressive.md algorithm_comparison 8 +``` + +### **Domain-Specific Networks:** +Focus on specific data domains by modifying the spec: +```bash +# Ocean/marine data network +> /project:infinite-web specs/ocean_network_progressive.md ocean_viz 10 + +# Climate change network +> /project:infinite-web specs/climate_network_progressive.md climate_viz 15 +``` + +### **Real-Time Dashboards:** +Advanced iterations include streaming data: +```bash +# Generate real-time monitoring dashboards +> /project:infinite-web specs/sdg_network_progressive.md realtime_sdg 20 +``` + +## Troubleshooting + +### **API Access Issues:** +- Some APIs require keys → iterations handle gracefully with fallbacks +- CORS errors → iterations use proxy or alternatives +- Rate limits → iterations implement caching + +### **Performance:** +- Large graphs → iterations use Canvas or WebGL +- Slow rendering → iterations implement LOD and culling +- Memory issues → iterations use virtualization + +### **Browser Compatibility:** +- All iterations target modern browsers (Chrome, Firefox, Safari, Edge) +- Fallbacks for older browsers where reasonable +- Progressive enhancement ensures basic functionality + +## Next Steps + +1. **Run your first iteration:** + ```bash + claude + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 + ``` + +2. **Review the output:** + - Open `sdg_viz/sdg_viz_1.html` in browser + - Check data sources and web learning attribution + - Test interactivity + +3. **Generate a batch:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 + ``` + +4. **Compare iterations:** + - See progressive improvements + - Note new techniques applied + - Observe API diversity + +5. **Go infinite:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite + ``` + +## Resources + +### **Specifications:** +- `specs/sdg_network_progressive.md` - Main visualization spec +- `specs/sdg_network_url_strategy.json` - Progressive learning URLs + +### **Commands:** +- `.claude/commands/infinite-web.md` - Web-enhanced loop implementation + +### **Baseline Reference:** +- `ai_docs/sdg-network-modelling/` - Original SDG network project + +### **Generated Output:** +- `sdg_viz/` - Output directory for iterations (create via command) + +## Contributing + +To extend this system: + +1. **Add new API sources** to `sdg_network_url_strategy.json` search templates +2. **Add learning resources** as new URLs in the strategy file +3. **Create domain-specific specs** by copying and modifying the base spec +4. **Share successful iterations** as examples + +## Credits + +- **Original SDG Network**: First Green Bank Network visualization +- **D3.js**: Michael Bostock and contributors +- **Observable**: Interactive notebook platform +- **Open APIs**: UN, World Bank, NOAA, USGS, NASA, and many others +- **Claude Code**: Infinite agentic loop implementation diff --git a/sdg_viz/sdg_viz_1.html b/sdg_viz/sdg_viz_1.html new file mode 100644 index 0000000..d51e8fe --- /dev/null +++ b/sdg_viz/sdg_viz_1.html @@ -0,0 +1,431 @@ + + + + + + SDG Network Visualization - Force-Directed Graph + + + + +
+
Loading UN SDG Data...
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/sdg_viz/sdg_viz_2.html b/sdg_viz/sdg_viz_2.html new file mode 100644 index 0000000..b70cf79 --- /dev/null +++ b/sdg_viz/sdg_viz_2.html @@ -0,0 +1,665 @@ + + + + + + SDG Network Viz 2: Environmental Indicators Network + + + + +
+
+

Environmental Indicators Network

+

Climate & Environment Data from World Bank Open Data API

+
+ +
+ + + + + + + +
+ +
+
+ + Loading environmental data from World Bank API... +
+
+ +
+

Legend

+
+

Node Colors (Indicator Categories)

+
+
+
+

Node Sizes (Data Magnitude)

+
+
+
+ Low values +
+
+
+ Medium values +
+
+
+ High values +
+
+
+
+

Edge Width (Correlation Strength)

+
+
+ + Weak correlation +
+
+ + Moderate correlation +
+
+ + Strong correlation +
+
+
+
+ +
+

Data Source & Learning Documentation

+

API Source: World Bank Open Data API (https://api.worldbank.org/v2/)

+

Indicators: CO2 Emissions (EN.ATM.CO2E.PC), Forest Area (AG.LND.FRST.ZS), Renewable Energy (EG.FEC.RNEW.ZS), Access to Electricity (EG.ELC.ACCS.ZS), PM2.5 Air Pollution (EN.ATM.PM25.MC.M3)

+

Web Learning Source: https://d3-graph-gallery.com/network.html

+

Techniques Applied:

+
    +
  • Color Scales: d3.scaleOrdinal() with d3.schemeCategory10 for categorical indicator types
    • +
  • Node Sizing: d3.scaleLinear() for proportional sizing based on data magnitude (5-25px range)
    • +
  • Force Simulation: Multi-body forces, link forces, collision detection, and centering
    • +
  • Visual Encodings: Size encodes value, color encodes category, edge width encodes correlation strength
    • +
  • Edge Styling: Curved paths with variable stroke-width based on relationship strength
    • +
  • Interactivity: Drag simulation, hover tooltips with rich data, zoom/pan behavior
    • +
+

Network Structure: Countries as nodes, environmental indicator correlations as edges. Nodes sized by normalized indicator values, colored by indicator category, connected when indicators show regional correlations.

+
+
+ +
+ + + + diff --git a/sdg_viz/sdg_viz_3.html b/sdg_viz/sdg_viz_3.html new file mode 100644 index 0000000..9459130 --- /dev/null +++ b/sdg_viz/sdg_viz_3.html @@ -0,0 +1,1014 @@ + + + + + + SDG Network Viz 3 - Global Biodiversity Interaction Network + + + + +
+
+

Interactive Controls

+ +
+

Search Species

+ + +
+ +
+

Filter by Taxonomic Rank

+
+
+ +
+

Actions

+ + + +
+ +
+

Node Details

+

Click a node to see details

+
+
+ +
+
+ + Loading biodiversity data from GBIF... +
+ +
+
+
+
+
+ +
+ Data Source: GBIF (Global Biodiversity Information Facility) API - Species occurrences and taxonomic hierarchy
+ Web Learning: D3 Temporal Force-Directed Graph
+ Techniques Applied: Advanced drag behavior with alpha targeting, dynamic graph updates with data joins, interactive node highlighting/focusing, search filtering, click/double-click event handling, smooth transitions, state management for selections +
+ + + + diff --git a/sdg_viz/sdg_viz_4.html b/sdg_viz/sdg_viz_4.html new file mode 100644 index 0000000..d3aa3e1 --- /dev/null +++ b/sdg_viz/sdg_viz_4.html @@ -0,0 +1,720 @@ + + + + + + SDG Network - Air Quality Monitoring (Performance Optimized) + + + + +
+
+

🌍 Global Air Quality Network

+

SDG 11: Sustainable Cities & Communities - Performance Optimized Edition

+
+ +
+
+
+ Initializing... +
+
+ Load time: -- +
+
+ +
+ + + +
+ +
+ +
+
Air Quality Categories
+
+
+
+ +
+
+
+
Cities Monitored
+
--
+
+
+
Network Connections
+
--
+
+
+
Avg PM2.5 (μg/m³)
+
--
+
+
+
Data Source
+
--
+
+
+
+ +
+
+ Performance Optimization Strategy:CACHED +
✅ localStorage caching with 24-hour TTL for instant subsequent loads +
✅ Embedded sample dataset for instant initial render (<100ms) +
✅ Background API fetch with progressive enhancement +
✅ Single optimized endpoint (OpenAQ API) with 5-second timeout +
+
+ Color Encoding Fix: +
✅ D3 categorical color scheme (d3.schemeTableau10) for distinct category colors +
✅ Proper ordinal scale mapping quality levels to unique colors +
✅ Clear visual legend showing all categories +
+
+ Web Learning Source: Observable D3 Color Schemes (https://observablehq.com/@d3/color-schemes) +
Techniques Applied: Categorical palettes (Tableau10), ordinal scale usage, color accessibility +
+
+ Iteration: sdg_viz_4.html | Generated: 2025-10-09 | Focus: Data Caching & Performance +
+
+
+ + + + \ No newline at end of file diff --git a/sdg_viz/sdg_viz_5.html b/sdg_viz/sdg_viz_5.html new file mode 100644 index 0000000..fbcaf99 --- /dev/null +++ b/sdg_viz/sdg_viz_5.html @@ -0,0 +1,739 @@ + + + + + + SDG Network Viz 5 - Advanced Color Encodings & Visual Hierarchy + + + + +
+
+

SDG Network Visualization - Advanced Color Encodings

+

Multi-Dimensional Color Strategy with Visual Hierarchy

+
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+ +
+ +
+ +
+
+
+ 🎨 + SDG Goal Categories (Click to Filter) +
+
+
+ +
+
+ 📊 + Impact Magnitude (Border Color & Size) +
+
+
+
+
+ Low Impact + Medium Impact + High Impact +
+
+
+
+ +
+
+ 💫 + Data Quality (Opacity) +
+
+
+
+
+ Low Quality (40%) +
+
+
+ Medium Quality (70%) +
+
+
+ High Quality (100%) +
+
+
+
+
+ +
+

Advanced Color Encoding Strategy

+ +

Multi-Attribute Encoding:

+
    +
  • Node Fill Color: SDG category (categorical - d3.schemeCategory10 default)
    • +
  • Node Border Color: Impact magnitude (sequential - d3.interpolateViridis default)
    • +
  • Node Opacity: Data quality/confidence (0.4-1.0 range)
    • +
  • Node Size: Impact magnitude (radius 8-28px)
    • +
  • Edge Color: Dynamic gradients from source to target node colors
    • +
  • Edge Opacity: Connection strength (0.2-0.8 range)
    • +
+ +

Web Learning Source: Observable HQ - D3 Color Legend (@d3/color-legend)

+

Techniques Applied:

+
    +
  • Perceptually uniform color scales (Viridis, Plasma, Cividis) for sequential encoding
    • +
  • Categorical color schemes (Category10, Accent, Dark2) for SDG categories
    • +
  • Custom interactive legends with click-to-filter functionality
    • +
  • Gradient bars for continuous scale visualization
    • +
  • Multi-dimensional encoding: fill + border + opacity + size
    • +
  • Dynamic color scheme switching for real-time exploration
    • +
+ +

Data Source: Embedded SDG network data (17 goals, 45+ connections) - Zero API latency

+ +
+
+
Color Dimensions
+
4 Attributes
+
+
+
Color Schemes
+
8+ Options
+
+
+
Interactive Elements
+
Legends + Filters
+
+
+
Load Time
+
<1 Second
+
+
+ +

+ SDG Network Visualization - Iteration 5 | Generated with web-enhanced learning | + Demonstrates advanced D3.js color encoding strategies +

+
+
+ + + + \ No newline at end of file diff --git a/sdg_viz/sdg_viz_6.html b/sdg_viz/sdg_viz_6.html new file mode 100644 index 0000000..214ee36 --- /dev/null +++ b/sdg_viz/sdg_viz_6.html @@ -0,0 +1,974 @@ + + + + + + SDG Network Viz 6 - ETL Pipeline with Embedded Multi-Year Data + + + + +
+ + +
+ +
+
+
+
+ +
+ ETL PIPELINE + Data Source: World Bank Climate Change API (Simulated) - CO₂ emissions, renewable energy, temperature anomalies
+ ETL Process: Data extracted 2025-10-09, cleaned & normalized, network metrics pre-computed (degree, betweenness, clustering), layout positions stabilized, statistical aggregations embedded
+ Web Learning: Observable: Introduction to Data
+ Techniques Applied: Pre-computation of network metrics, embedded multi-year data snapshots for instant switching, efficient JSON storage format, data transformation pipelines (cleaning, normalization, enrichment), statistical summary calculations (mean, median, std dev), histogram visualization of degree distribution, data export functionality, zero API calls for sub-500ms load time +
+ + + + diff --git a/sdg_viz/sdg_viz_7.html b/sdg_viz/sdg_viz_7.html new file mode 100644 index 0000000..8e519cd --- /dev/null +++ b/sdg_viz/sdg_viz_7.html @@ -0,0 +1,904 @@ + + + + + + SDG Network - Exploratory Analysis: Brushing & Linking + + + + +
+

SDG Climate Network - Exploratory Analysis

+
Brushing & Linking for Multi-View Coordination
+
+ +
+ + +
+ +
+
+
+ Force Network View + 0 selected +
+ +
+ +
+
+ Connection Degree Distribution + 0 selected +
+ +
+ +
+
+ Impact vs Connectivity + 0 selected +
+ +
+
+ +
+ +
+ Iteration 7: Exploratory Analysis with Brushing & Linking
+ This visualization demonstrates advanced exploratory data analysis using coordinated multiple views. + Rectangle brush on the network to select nodes, and observe synchronized highlighting across all views. + Features d3.brush() for selection, coordinated view updates, and multi-select capability (Shift+Click).
+ Web Learning: Techniques from D3 Brushable Scatterplot - + Applied brush event handling, selection propagation via dispatch, coordinated filtering across views, and synchronized visual feedback.
+ Data: Embedded climate action network (50 nodes, 85 connections) - Instant load, offline-ready +
+ + + + diff --git a/sdg_viz/sdg_viz_8.html b/sdg_viz/sdg_viz_8.html new file mode 100644 index 0000000..93251e2 --- /dev/null +++ b/sdg_viz/sdg_viz_8.html @@ -0,0 +1,970 @@ + + + + + + SDG Network Viz 8 - Hierarchical SDG Taxonomy with Multi-Layout Transitions + + + + +
+
+

Hierarchical Controls

+ +
+

Layout Mode

+ + + + +
+ +
+

Hierarchy Actions

+ + + + +
+ +
+

Node Information

+

Click a node to view details
Click again to expand/collapse children

+
+
+ +
+ +
+
+
+
+
+ +
+ Iteration 8 - Advanced D3: Hierarchical Networks & Transitions
+ Web Source: D3 Force-Directed Tree
+ Techniques Applied: d3.hierarchy() for hierarchical data processing, force simulation with link/charge/center forces, collapsible nodes (click to expand/collapse), multiple layout algorithms (force/tree/radial/cluster), smooth d3.transition() animations (500ms, easeQuadInOut), state management with URL parameters, zoom-to-fit functionality, hierarchical parent-child link constraints
+ Hierarchy: SDG Goals → Targets → Indicators (3-level taxonomy), embedded data for instant load +
+ + + + diff --git a/sdg_viz/sdg_viz_9.html b/sdg_viz/sdg_viz_9.html new file mode 100644 index 0000000..f5a5f1e --- /dev/null +++ b/sdg_viz/sdg_viz_9.html @@ -0,0 +1,916 @@ + + + + + + SDG Network Visualization 9 - High Performance Canvas Rendering + + + + +
+ + +
+

⚙️ Performance Controls

+ +
+ +
+ + +
+
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+

📊 Performance Metrics

+
+ FPS: + 60 +
+
+ Visible Nodes: + 0 +
+
+ Total Nodes: + 0 +
+
+ Render Time: + 0ms +
+
+ Simulation: + Running +
+
+ Quadtree Depth: + 0 +
+
+ +
+
Loading nodes progressively...
+
+
+
+
+ +
+

SDG Categories

+
+ +
+
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/specs/sdg_network_progressive.md b/specs/sdg_network_progressive.md new file mode 100644 index 0000000..8fb7b79 --- /dev/null +++ b/specs/sdg_network_progressive.md @@ -0,0 +1,446 @@ +# Progressive SDG Network Visualization Specification + +## Core Challenge +Create **progressively sophisticated SDG network visualizations** that integrate multiple open data APIs to display environmental, scientific, and sustainable development data as interactive force-directed graphs. Each iteration should discover new data sources and enhance visual/interactive capabilities. + +## Output Requirements + +**File Naming**: `sdg_viz_[iteration_number].html` + +**Content Structure**: Self-contained HTML file with embedded D3.js visualization +```html + + + + + + SDG Network Visualization [Iteration Number] - [Data Source Theme] + + + +
+ + + + +
+ +
+ + +
+ +
+ + +
+ +
+
+ + + + + + + + + + +``` + +## Progressive Enhancement Dimensions + +### **Phase 1: Foundation (Iterations 1-5)** +**Focus**: Basic network visualization with single API integration + +**Node Enhancements:** +- Size based on data properties (e.g., population, magnitude) +- Color coding by category/type (using color scales) +- Clear labels with smart positioning +- Hover tooltips with basic information + +**Edge Enhancements:** +- Width based on relationship strength +- Color coding for relationship types +- Straight lines with arrows for directionality +- Dashed/solid patterns for different relationship classes + +**Interactivity:** +- Drag nodes to reposition +- Zoom and pan controls +- Click to highlight connected nodes +- Basic filtering by node type + +**Data Integration:** +- Single API source per iteration +- Simple JSON data transformation +- Basic node-edge relationship mapping +- Manual API endpoint definition + +### **Phase 2: Intermediate (Iterations 6-12)** +**Focus**: Multi-source integration and enhanced interactivity + +**Node Enhancements:** +- Dynamic sizing with min/max constraints +- Multi-property color encoding (fill + border) +- Icon/symbol overlays for node types +- Rich tooltips with formatted data tables +- Node clustering and grouping +- Custom shapes for different entity types + +**Edge Enhancements:** +- Curved edges for better visibility +- Dynamic edge strength from data properties +- Animated flow along edges +- Edge bundling for cleaner layouts +- Multiple edge types with different styles + +**Interactivity:** +- Search functionality to find nodes +- Multi-select for comparison +- Time-based filtering/animation +- Expandable/collapsible node groups +- Side panel with detailed statistics +- Export data/visualization options + +**Data Integration:** +- Combine 2-3 API sources +- Automatic data merging by key fields +- API discovery via Swagger/OpenAPI specs +- Error handling for API failures +- Caching for performance + +### **Phase 3: Advanced (Iterations 13-20)** +**Focus**: Sophisticated algorithms and real-time capabilities + +**Node Enhancements:** +- Force-based clustering algorithms +- Community detection visualization +- Centrality measures (betweenness, closeness) +- Node importance ranking +- Multi-level node hierarchies +- Animated node state transitions + +**Edge Enhancements:** +- Weighted force simulation +- Edge confidence/uncertainty visualization +- Temporal edge evolution +- Hierarchical edge bundling +- Edge crossing minimization + +**Interactivity:** +- Real-time data updates +- Graph layout algorithm selection +- Advanced filtering with boolean logic +- Shortest path highlighting +- Neighborhood exploration +- Graph metric dashboards +- Collaborative filtering + +**Data Integration:** +- 4+ API sources integrated +- Automated API endpoint discovery +- Data quality indicators +- Cross-API entity resolution +- Streaming data integration +- Webhook support for updates + +### **Phase 4: Expert (Iterations 21+)** +**Focus**: Novel techniques and AI-enhanced features + +**Node Enhancements:** +- ML-based node positioning +- Anomaly detection highlighting +- Predictive node attributes +- Semantic node embedding +- Dynamic node importance +- Multi-dimensional projection + +**Edge Enhancements:** +- Probabilistic edge rendering +- Predicted relationship suggestions +- Edge importance scoring +- Dynamic edge weight learning +- Causal relationship inference + +**Interactivity:** +- Natural language queries +- AI-powered graph insights +- Automatic pattern detection +- Interactive graph queries (Cypher-like) +- Graph comparison tools +- VR/AR integration capabilities + +**Data Integration:** +- Federated query across APIs +- Automatic schema alignment +- Knowledge graph construction +- Semantic web integration (RDF, SPARQL) +- Real-time distributed data +- Blockchain data sources + +## API Discovery Strategy + +### **Target Data Domains** +Each iteration should discover and integrate APIs from these domains: + +**Environmental Data:** +- Air quality (OpenAQ, EPA APIs) +- Water quality (USGS, EPA Water) +- Climate data (NOAA, NASA) +- Weather (OpenWeatherMap, Weather.gov) +- Ocean data (NOAA Ocean, Marine APIs) + +**Scientific Data:** +- Biodiversity (GBIF, iNaturalist) +- Forest data (Global Forest Watch) +- Soil data (SoilGrids, ISRIC) +- Satellite imagery (NASA Earth, Sentinel) +- Seismic data (USGS Earthquake) + +**Sustainable Development:** +- UN SDG API (existing baseline) +- World Bank Open Data +- OECD Statistics +- WHO Global Health +- FAO Agricultural Data + +**Discovery Process:** +1. **Web Search for APIs**: Use WebSearch to find: `"[domain] API swagger site:github.com"` +2. **Swagger/OpenAPI Analysis**: Parse API specs to find endpoints +3. **Data Quality Check**: Verify API has relevant network-ready data +4. **Integration Planning**: Map API data to nodes and edges +5. **Documentation**: Record API details in visualization + +### **API Selection Criteria** +- **Open Access**: Free or freemium tier available +- **REST/JSON**: Easy integration with JavaScript +- **Documentation**: Well-documented endpoints +- **Reliability**: Active maintenance and uptime +- **Data Richness**: Complex enough for network visualization +- **SDG Relevance**: Connects to sustainability goals + +## Network Data Model + +### **Node Schema** +```javascript +{ + id: "unique_identifier", + name: "Display Name", + type: "category", // e.g., "country", "indicator", "species" + group: "group_name", // for color coding + size: numeric_value, // for visual sizing + properties: { + // Domain-specific attributes + value: number, + unit: "string", + timestamp: "ISO_date", + source_api: "api_name", + // ... additional properties + }, + x: number, // force layout position + y: number +} +``` + +### **Edge Schema** +```javascript +{ + source: "node_id", + target: "node_id", + type: "relationship_type", + strength: numeric_value, // for force simulation + weight: numeric_value, // for visual width + properties: { + correlation: number, + confidence: number, + source_api: "api_name", + // ... additional properties + } +} +``` + +## Web Learning Integration + +### **Each Iteration Must:** +1. **Fetch Assigned URL**: Use WebFetch to retrieve documentation/tutorial +2. **Extract Techniques**: Identify 1-3 specific techniques from the source +3. **Apply Learning**: Implement the technique in the visualization +4. **Document Source**: Clearly attribute the web source and what was learned +5. **Demonstrate Improvement**: Show measurable enhancement from previous iteration + +### **Web Source Categories** + +**D3.js Techniques:** +- Observable HQ notebooks (d3.js examples) +- D3.js official documentation +- Mike Bostock's blocks +- D3 Graph Gallery +- Interactive graph tutorials + +**Force-Directed Graphs:** +- Force simulation parameters +- Collision detection +- Link distance functions +- Clustering algorithms +- Layout optimization + +**Data Integration:** +- REST API patterns +- CORS handling +- Async data loading +- Data transformation pipelines +- Error handling strategies + +**Visual Design:** +- Color theory for networks +- Graph readability principles +- Interactive UX patterns +- Animation best practices +- Accessibility in visualizations + +## Quality Standards + +### **Functional Requirements** +- ✅ Fetches data from at least one open API +- ✅ Renders as force-directed graph with D3.js +- ✅ Nodes and edges have meaningful properties from data +- ✅ Interactive (hover, click, drag minimum) +- ✅ Responsive design works on desktop and mobile +- ✅ Clear data source attribution +- ✅ Error handling for API failures + +### **Visual Excellence** +- ✅ Professional color scheme with good contrast +- ✅ Readable labels and legends +- ✅ Smooth animations (60fps) +- ✅ Clear visual hierarchy +- ✅ Consistent styling throughout +- ✅ Loading states for async operations + +### **Technical Quality** +- ✅ Self-contained HTML file (or minimal external deps) +- ✅ Clean, commented JavaScript code +- ✅ Efficient force simulation (no lag) +- ✅ Proper error handling and fallbacks +- ✅ Browser compatibility (modern browsers) +- ✅ No console errors + +### **Progressive Learning** +- ✅ Demonstrates new technique from web source +- ✅ Builds upon previous iterations where appropriate +- ✅ Documents what was learned and how applied +- ✅ Shows measurable improvement +- ✅ Explores new API sources + +## Iteration Evolution Strategy + +### **Starting Point (Iteration 1)** +- Use UN SDG API (baseline from existing project) +- Basic force-directed graph with D3.js v7 +- Simple node/edge styling +- Basic hover tooltips +- Learn from: D3.js force simulation documentation + +### **Early Iterations (2-5)** +- Add one new API per iteration +- Focus on core graph enhancements +- Build interactive controls +- Learn: Color scales, tooltips, filtering + +### **Middle Iterations (6-12)** +- Combine multiple APIs +- Advanced interactivity +- Custom layouts and algorithms +- Learn: Graph algorithms, data merging, UX patterns + +### **Advanced Iterations (13-20)** +- Real-time data integration +- Complex algorithms (community detection, centrality) +- Rich dashboards +- Learn: Advanced D3, graph theory, streaming data + +### **Expert Iterations (21+)** +- AI/ML integration +- Novel visualization techniques +- Cross-platform capabilities +- Learn: Cutting-edge graph visualization research + +## Success Metrics + +Each iteration should improve on at least 3 of these metrics: + +**Data Richness:** +- Number of API sources integrated +- Total nodes in network +- Total edges in network +- Data update frequency + +**Visual Quality:** +- Color coding sophistication +- Label readability score +- Animation smoothness +- Layout clarity + +**Interactivity:** +- Number of interactive features +- Response time to user actions +- Filter/search capabilities +- Information density + +**Technical Innovation:** +- New techniques applied +- Performance improvements +- Code quality metrics +- Browser compatibility + +## Ultra-Thinking Directive + +Before each iteration, deeply consider: + +**API Discovery:** +- What domain provides the most value for SDG insights? +- Which APIs have network-ready data structures? +- How can multiple APIs be meaningfully combined? +- What data relationships would be most revealing? + +**Visualization Strategy:** +- What graph layout best reveals the data patterns? +- How can node/edge properties enhance understanding? +- What interactivity would provide the most insight? +- How does this build on previous iterations? + +**Web Learning Application:** +- What specific technique from the web source applies here? +- How can this technique improve the visualization? +- What adaptations are needed for our use case? +- How does this advance the state of the art? + +**User Value:** +- What insights does this visualization reveal? +- How intuitive is the interaction model? +- What questions can users answer with this tool? +- How does this serve SDG analysis and understanding? + +**Generate visualizations that:** +- **Discover New Data**: Find and integrate novel API sources +- **Apply Web Knowledge**: Demonstrate clear learning from web research +- **Enhance Progressively**: Each iteration measurably better than the last +- **Serve SDGs**: Provide genuine value for sustainable development insights +- **Inspire Exploration**: Create compelling, informative, interactive experiences diff --git a/specs/sdg_network_url_strategy.json b/specs/sdg_network_url_strategy.json new file mode 100644 index 0000000..1b633aa --- /dev/null +++ b/specs/sdg_network_url_strategy.json @@ -0,0 +1,278 @@ +{ + "description": "Progressive web learning strategy for SDG network visualizations with API integration", + "categories": { + "foundation": { + "description": "Basic D3.js force-directed graphs and API integration (Iterations 1-5)", + "urls": [ + { + "url": "https://d3js.org/what-is-d3", + "topic": "D3.js fundamentals and data binding", + "techniques": ["data binding", "selections", "enter/update/exit pattern"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph", + "topic": "Basic force-directed graph implementation", + "techniques": ["force simulation", "node positioning", "link rendering"] + }, + { + "url": "https://d3-graph-gallery.com/network.html", + "topic": "Network visualization gallery and examples", + "techniques": ["color scales", "node sizing", "edge styling"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph-canvas", + "topic": "Canvas-based force graphs for performance", + "techniques": ["canvas rendering", "performance optimization", "event handling"] + }, + { + "url": "https://observablehq.com/@d3/temporal-force-directed-graph", + "topic": "Adding interactivity to force graphs", + "techniques": ["drag behavior", "zoom", "tooltip interactions"] + } + ] + }, + "intermediate": { + "description": "Advanced D3 techniques and multi-API integration (Iterations 6-12)", + "urls": [ + { + "url": "https://observablehq.com/@d3/sticky-force-layout", + "topic": "Sticky force layout for better control", + "techniques": ["fixed positions", "sticky drag", "layout constraints"] + }, + { + "url": "https://observablehq.com/@d3/clustered-force-layout", + "topic": "Force-based clustering", + "techniques": ["cluster forces", "group positioning", "collision detection"] + }, + { + "url": "https://observablehq.com/@d3/arc-diagram", + "topic": "Alternative network layouts", + "techniques": ["arc diagrams", "hierarchical layouts", "radial layouts"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-lattice", + "topic": "Custom force implementations", + "techniques": ["custom forces", "lattice constraints", "force composition"] + }, + { + "url": "https://observablehq.com/@d3/color-schemes", + "topic": "Advanced color schemes for networks", + "techniques": ["sequential scales", "diverging scales", "categorical colors"] + }, + { + "url": "https://observablehq.com/@d3/connected-particles", + "topic": "Animated network transitions", + "techniques": ["particle systems", "edge animation", "smooth transitions"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph-component", + "topic": "Modular graph components", + "techniques": ["component architecture", "reusable patterns", "state management"] + } + ] + }, + "advanced": { + "description": "Complex algorithms and real-time data (Iterations 13-20)", + "urls": [ + { + "url": "https://observablehq.com/@d3/hierarchical-edge-bundling", + "topic": "Edge bundling for clarity", + "techniques": ["hierarchical bundling", "edge aggregation", "path smoothing"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-tree", + "topic": "Force-directed trees and hierarchies", + "techniques": ["tree forces", "hierarchy layout", "parent-child links"] + }, + { + "url": "https://observablehq.com/@mbostock/adaptive-sampling", + "topic": "Adaptive rendering for large graphs", + "techniques": ["level of detail", "culling", "adaptive sampling"] + }, + { + "url": "https://observablehq.com/@d3/versor-dragging", + "topic": "3D network rotations", + "techniques": ["quaternion rotation", "3D projection", "spherical layout"] + }, + { + "url": "https://observablehq.com/@d3/voronoi-labels", + "topic": "Smart label placement", + "techniques": ["voronoi diagrams", "collision-free labels", "dynamic positioning"] + }, + { + "url": "https://observablehq.com/@d3/zoom-to-bounding-box", + "topic": "Advanced zoom and focus", + "techniques": ["semantic zoom", "focus+context", "smooth camera transitions"] + }, + { + "url": "https://observablehq.com/@mbostock/the-wealth-health-of-nations", + "topic": "Real-time data integration", + "techniques": ["data streaming", "time series", "dynamic updates"] + }, + { + "url": "https://observablehq.com/@d3/graph-layout-force-directed", + "topic": "Graph layout algorithms comparison", + "techniques": ["layout selection", "parameter tuning", "performance trade-offs"] + } + ] + }, + "expert": { + "description": "Cutting-edge techniques and AI integration (Iterations 21+)", + "urls": [ + { + "url": "https://observablehq.com/@d3/force-directed-graph-with-minimap", + "topic": "Multi-view graph interfaces", + "techniques": ["minimap", "multiple synchronized views", "overview+detail"] + }, + { + "url": "https://observablehq.com/@mbostock/phase-portrait", + "topic": "Dynamic system visualization", + "techniques": ["phase space", "attractor visualization", "vector fields"] + }, + { + "url": "https://observablehq.com/@fil/occlusion-hello-world", + "topic": "3D occlusion and depth", + "techniques": ["depth sorting", "occlusion culling", "3D rendering"] + }, + { + "url": "https://observablehq.com/@d3/graph", + "topic": "Graph theory algorithms", + "techniques": ["shortest path", "centrality measures", "community detection"] + }, + { + "url": "https://observablehq.com/@mbostock/saving-svg", + "topic": "Export and sharing capabilities", + "techniques": ["SVG export", "image generation", "data export"] + } + ] + }, + "api_discovery": { + "description": "Web search templates for finding open APIs", + "search_templates": [ + { + "query": "environmental data API swagger site:github.com", + "domain": "Environmental APIs", + "expected_sources": ["air quality", "water quality", "climate data"] + }, + { + "query": "scientific data REST API openapi", + "domain": "Scientific Data", + "expected_sources": ["biodiversity", "satellite", "research datasets"] + }, + { + "query": "sustainable development goals API JSON", + "domain": "SDG Data", + "expected_sources": ["UN data", "World Bank", "development indicators"] + }, + { + "query": "atmospheric weather data API free", + "domain": "Atmospheric Data", + "expected_sources": ["weather", "air quality", "climate"] + }, + { + "query": "water quality watershed API REST", + "domain": "Water Data", + "expected_sources": ["USGS", "EPA", "water monitoring"] + }, + { + "query": "forest biodiversity data API open source", + "domain": "Ecological Data", + "expected_sources": ["Global Forest Watch", "species data", "conservation"] + }, + { + "query": "NOAA ocean marine data API", + "domain": "Ocean Data", + "expected_sources": ["ocean temperature", "marine life", "coastal data"] + }, + { + "query": "NASA earth observation API", + "domain": "Satellite Data", + "expected_sources": ["satellite imagery", "earth science", "remote sensing"] + }, + { + "query": "seismic earthquake API USGS", + "domain": "Geophysical Data", + "expected_sources": ["earthquake", "volcano", "geological"] + }, + { + "query": "WHO global health statistics API", + "domain": "Health Data", + "expected_sources": ["disease", "healthcare", "mortality"] + } + ], + "api_evaluation_criteria": [ + "Has Swagger/OpenAPI specification", + "Provides JSON/REST endpoints", + "Free tier or open access available", + "Good documentation", + "Active maintenance (updated recently)", + "Network-ready data (entities with relationships)", + "Relevant to SDGs or sustainability" + ] + }, + "data_integration": { + "description": "Resources for API integration and data merging", + "urls": [ + { + "url": "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch", + "topic": "Modern fetch API usage", + "techniques": ["async/await", "error handling", "CORS"] + }, + { + "url": "https://observablehq.com/@observablehq/introduction-to-data", + "topic": "Data loading and transformation", + "techniques": ["data parsing", "cleaning", "transformation pipelines"] + }, + { + "url": "https://observablehq.com/@d3/d3-fetch", + "topic": "D3 data fetching utilities", + "techniques": ["d3.json", "d3.csv", "data type inference"] + } + ] + }, + "graph_algorithms": { + "description": "Graph theory and network analysis resources", + "urls": [ + { + "url": "https://observablehq.com/@d3/disjoint-force-directed-graph", + "topic": "Disconnected graph handling", + "techniques": ["component detection", "separate force simulations"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph?intent=fork", + "topic": "Customizable force parameters", + "techniques": ["force strength tuning", "link distance optimization"] + }, + { + "url": "https://observablehq.com/@mbostock/paths-in-a-force-directed-graph", + "topic": "Path finding and highlighting", + "techniques": ["shortest path", "path visualization", "connected components"] + } + ] + } + }, + "iteration_mapping": { + "1-5": "foundation", + "6-12": "intermediate", + "13-20": "advanced", + "21+": "expert" + }, + "fallback_strategy": { + "description": "When pre-defined URLs are exhausted, use dynamic web search", + "search_patterns": [ + "D3.js force directed graph [specific_technique]", + "network visualization [technique] observable", + "graph layout algorithm [specific_algorithm]", + "[api_domain] open API swagger github", + "force simulation [parameter] tuning d3" + ] + }, + "priming_urls": { + "description": "Initial URLs to fetch before starting any iteration for foundational knowledge", + "urls": [ + "https://d3js.org/what-is-d3", + "https://observablehq.com/@d3/force-directed-graph", + "https://github.com/public-apis/public-apis" + ] + } +}