diff --git a/.claude/commands/write.md b/.claude/commands/write.md new file mode 100644 index 0000000..0ea381d --- /dev/null +++ b/.claude/commands/write.md @@ -0,0 +1 @@ +OK can you create a manual in @ai_docs/ that explains everything that we accomplished today? diff --git a/SDG_NETWORK_ANALYSIS.md b/SDG_NETWORK_ANALYSIS.md new file mode 100644 index 0000000..8f7c4d4 --- /dev/null +++ b/SDG_NETWORK_ANALYSIS.md @@ -0,0 +1,540 @@ +# SDG Network Visualization - Analysis & Strategy + +## Executive Summary + +I've analyzed the existing SDG network modelling project and designed a **web-enhanced infinite loop system** that will progressively generate sophisticated force-directed network visualizations with automatic API discovery. + +### What Was Reviewed + +**Existing SDG Network Project** (`ai_docs/sdg-network-modelling/`): +- Python notebooks fetching UN SDG API data +- D3.js v3 force-directed graph visualization +- Basic network showing: Geographic areas ↔ Ecosystem Capitals ↔ SDGs ↔ Targets +- Simple interactions: hover tooltips, drag nodes +- Single API source (UN Statistics) + +**Current Limitations:** +- Outdated D3.js version (v3 vs current v7) +- Single hardcoded API +- Basic visual design (simple circles/lines) +- Limited interactivity (just hover) +- No progressive enhancement +- No web learning integration + +## New System Design + +### Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Web-Enhanced Infinite Loop (/project:infinite-web) │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Phase 0: │ → │ Phase 1: │ → │ Phase 2: │ │ +│ │ Web Priming │ │ Spec + Web │ │ API │ │ +│ │ │ │ Analysis │ │ Discovery │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ ↓ ↓ ↓ │ +│ Fetch D3.js docs Read spec file Search for APIs │ +│ Learn force graphs + priming knowledge Parse Swagger │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Phase 3: │ → │ Phase 4: │ → │ Phase 5: │ │ +│ │ Iteration │ │ Parallel │ │ Quality │ │ +│ │ Strategy │ │ Agents │ │ Assurance │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ ↓ ↓ ↓ │ +│ Map URLs to iters Launch sub-agents Verify outputs │ +│ Plan API sources Each learns + builds Check API usage │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Progressive Enhancement Strategy + +**Phase 1: Foundation (Iterations 1-5)** +- Modern D3.js v7 +- Single API per iteration +- Node sizing, color coding, labels +- Basic interactivity (drag, zoom, click) +- Learn from: D3.js fundamentals, force simulation basics + +**Phase 2: Intermediate (Iterations 6-12)** +- Combine 2-3 APIs +- Multi-property encoding (color + border) +- Edge bundling, curved edges +- Search, filter, side panels +- Learn from: Advanced D3 techniques, data integration + +**Phase 3: Advanced (Iterations 13-20)** +- 4+ API sources +- Clustering algorithms, community detection +- Real-time data updates +- Graph metrics dashboards +- Learn from: Graph algorithms, streaming data + +**Phase 4: Expert (Iterations 21+)** +- Federated API queries +- ML-based layouts +- AI-powered insights +- 3D/VR capabilities +- Learn from: Cutting-edge research, novel techniques + +## API Discovery System + +### Automatic Discovery Process + +Each iteration performs web searches to find new APIs: + +```javascript +// Search patterns (from sdg_network_url_strategy.json) +"environmental data API swagger site:github.com" +"scientific data REST API openapi" +"sustainable development goals API JSON" +"atmospheric weather data API free" +"water quality watershed API REST" +"forest biodiversity data API open source" +"NOAA ocean marine data API" +"NASA earth observation API" +``` + +### Target API Domains + +**Environmental:** +- OpenAQ (air quality) +- EPA APIs (environmental protection) +- USGS Water Services +- Global Forest Watch + +**Scientific:** +- GBIF (biodiversity) +- NASA Earthdata +- NOAA (ocean, weather, climate) +- Seismic/earthquake data (USGS) + +**Sustainable Development:** +- UN SDG API (baseline) +- World Bank Open Data +- OECD Statistics +- WHO Global Health +- FAO Agricultural Data + +**Data Integration:** +- Swagger/OpenAPI spec parsing +- Automatic entity mapping to nodes/edges +- Cross-API entity resolution +- Data quality assessment + +## Network Data Model + +### Node Structure +```javascript +{ + id: "unique_id", + name: "Display Name", + type: "entity_type", // country, indicator, species, etc. + group: "category", // for color coding + size: value, // visual size from data + properties: { + value: number, + unit: "string", + timestamp: "date", + source_api: "api_name", + // ... domain-specific attributes + } +} +``` + +### Edge Structure +```javascript +{ + source: "node_id", + target: "node_id", + type: "relationship", // correlation, dependency, etc. + strength: value, // for force simulation + weight: value, // visual width + properties: { + correlation: number, + confidence: number, + source_api: "api_name" + } +} +``` + +## Visual Enhancement Progression + +### Node Enhancements +**Foundation:** +- Size ← data magnitude +- Color ← category +- Labels with smart positioning + +**Intermediate:** +- Border color ← secondary property +- Icon overlays ← entity type +- Clustering ← community detection + +**Advanced:** +- Shape ← entity class +- Opacity ← confidence level +- Animation ← state changes + +**Expert:** +- 3D projection ← multi-dimensional data +- ML-based positioning +- Anomaly highlighting + +### Edge Enhancements +**Foundation:** +- Width ← relationship strength +- Color ← relationship type +- Arrows ← directionality + +**Intermediate:** +- Curved paths ← visibility +- Bundling ← clarity +- Dash patterns ← relationship class + +**Advanced:** +- Animated flow ← data transfer +- Dynamic strength ← real-time values +- Hierarchical bundling ← complex relationships + +**Expert:** +- Probabilistic rendering +- Predicted relationships +- Causal inference visualization + +## Web Learning Integration + +### URL Strategy Structure + +The system uses `specs/sdg_network_url_strategy.json` with: + +**Foundation URLs (Iterations 1-5):** +- D3.js fundamentals +- Basic force-directed graphs +- Color scales and node styling +- Interactive tooltips + +**Intermediate URLs (Iterations 6-12):** +- Force-based clustering +- Custom force implementations +- Advanced color schemes +- Animated transitions + +**Advanced URLs (Iterations 13-20):** +- Edge bundling algorithms +- 3D projections +- Adaptive rendering +- Real-time data integration + +**Expert URLs (Iterations 21+):** +- Graph theory algorithms +- ML-enhanced layouts +- Multi-view interfaces +- Export capabilities + +### Learning Process Per Iteration + +1. **WebFetch** assigned URL +2. **Extract** 1-3 specific techniques +3. **Implement** in visualization +4. **Document** source and learnings +5. **Demonstrate** improvement + +## File Structure + +### Created Files + +``` +specs/ +├── sdg_network_progressive.md # Main specification +└── sdg_network_url_strategy.json # Learning URLs + +SDG_NETWORK_GUIDE.md # Complete usage guide +SDG_NETWORK_ANALYSIS.md # This file (analysis) +CLAUDE.md # Updated with SDG commands +``` + +### Output Structure + +``` +sdg_viz/ # Created by command +├── sdg_viz_1.html # Iteration 1 +├── sdg_viz_2.html # Iteration 2 +├── ... +└── sdg_viz_N.html # Iteration N +``` + +Each HTML file is self-contained with: +- D3.js v7 force-directed graph +- API data fetching +- Interactive controls +- Documentation footer (APIs used, web learning) + +## Usage Examples + +### Quick Start +```bash +claude +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 +``` + +Output: `sdg_viz/sdg_viz_1.html` +- UN SDG API baseline +- Basic D3.js force graph +- Foundation-level enhancements + +### Small Batch +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 +``` + +Output: 5 files, each with: +- Different API source +- Progressive D3 techniques +- Building on previous iterations + +### Full Progressive Learning +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 12 specs/sdg_network_url_strategy.json +``` + +Output: 12 iterations +- Iterations 1-5: Foundation (basic graphs, single APIs) +- Iterations 6-12: Intermediate (multi-API, advanced interactivity) + +### Infinite Mode +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite specs/sdg_network_url_strategy.json +``` + +Runs until context limit: +- Progressive difficulty waves +- Automatic API discovery +- Increasingly sophisticated visualizations + +## Key Innovations + +### 1. Automatic API Discovery +- **Old**: Hardcoded UN SDG API only +- **New**: Web searches find Swagger/OpenAPI endpoints +- **Impact**: Unlimited data source expansion + +### 2. Progressive Web Learning +- **Old**: Static implementation +- **New**: Each iteration learns from curated web resources +- **Impact**: Continuous technique improvement + +### 3. Multi-Source Integration +- **Old**: Single API +- **New**: Combine 2-5+ APIs per iteration +- **Impact**: Richer, more insightful networks + +### 4. Modern D3.js +- **Old**: D3.js v3 (2012) +- **New**: D3.js v7 (2023) +- **Impact**: Better performance, modern features + +### 5. Rich Interactivity +- **Old**: Basic hover tooltip +- **New**: Search, filter, zoom, click actions, exports, real-time updates +- **Impact**: Deeper data exploration + +## Quality Metrics + +Each iteration evaluated on: + +### Data Richness +- API sources count (1 → 5+) +- Node count +- Edge count +- Update frequency + +### Visual Quality +- Color sophistication +- Label readability +- Animation smoothness (60fps target) +- Layout clarity score + +### Interactivity +- Feature count +- Response time +- Filter capabilities +- Information density + +### Technical Innovation +- New techniques applied +- Performance metrics +- Code quality +- Browser compatibility + +## Comparison: Old vs New + +| Aspect | Old System | New System | +|--------|-----------|------------| +| **D3 Version** | v3 (2012) | v7 (2023) | +| **APIs** | 1 (hardcoded) | Unlimited (auto-discovery) | +| **Node Styling** | Basic circles | Size, color, border, icons, shapes | +| **Edge Styling** | Simple lines | Width, color, curves, bundling, animation | +| **Interactivity** | Hover only | Hover, click, drag, zoom, search, filter | +| **Learning** | None | Web-sourced techniques per iteration | +| **Data Sources** | UN SDG only | Environmental, scientific, SDG, health, etc. | +| **Architecture** | Static | Progressive enhancement | +| **Output** | Single file | Iteration series | + +## Next Steps + +### Immediate Actions + +1. **Test Single Iteration:** + ```bash + claude + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 + ``` + +2. **Review Output:** + - Open `sdg_viz/sdg_viz_1.html` in browser + - Check UN SDG API integration + - Test basic interactivity + - Verify web learning attribution + +3. **Generate Small Batch:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 + ``` + +4. **Compare Iterations:** + - See API diversity + - Note progressive improvements + - Identify new techniques applied + +### Advanced Exploration + +1. **Custom API Focus:** + - Modify spec to target specific domains (ocean, climate, etc.) + - Adjust search templates for specific APIs + +2. **Algorithm Comparison:** + - Generate iterations with different force algorithms + - Compare layout effectiveness + +3. **Real-time Dashboards:** + - Generate iterations 13-20 for real-time features + - Test streaming data integration + +4. **Infinite Mode:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite specs/sdg_network_url_strategy.json + ``` + +### Customization Options + +**Create Domain-Specific Specs:** +```bash +# Ocean/marine networks +cp specs/sdg_network_progressive.md specs/ocean_network.md +# Edit to focus on ocean APIs + +# Climate change networks +cp specs/sdg_network_progressive.md specs/climate_network.md +# Edit to focus on climate APIs +``` + +**Extend URL Strategy:** +- Add new D3 technique URLs +- Add domain-specific learning resources +- Create custom search templates + +**Modify Output Format:** +- Change from single HTML to multi-file +- Add server-side API proxies +- Generate dashboards with multiple views + +## Success Indicators + +### Technical Success +- ✅ Modern D3.js v7 implementation +- ✅ Multiple API sources per iteration +- ✅ Self-contained HTML files +- ✅ 60fps animations +- ✅ No console errors + +### Functional Success +- ✅ Force-directed graphs render correctly +- ✅ API data fetches successfully +- ✅ Interactivity works smoothly +- ✅ Visualizations are informative +- ✅ Error handling is graceful + +### Learning Success +- ✅ Each iteration demonstrates new technique +- ✅ Web sources are properly attributed +- ✅ Progressive improvement is evident +- ✅ API diversity increases +- ✅ Complexity grows appropriately + +### User Value +- ✅ Insights are actionable +- ✅ Exploration is intuitive +- ✅ Data relationships are clear +- ✅ SDG analysis is enhanced +- ✅ Visualizations are shareable + +## Troubleshooting Guide + +### API Access Issues +**Problem**: API requires authentication +**Solution**: Iterations handle gracefully with fallback APIs + +**Problem**: CORS errors +**Solution**: Use CORS proxies or alternative APIs + +**Problem**: Rate limiting +**Solution**: Implement caching and request throttling + +### Performance Issues +**Problem**: Large graphs lag +**Solution**: Canvas rendering (implemented in advanced iterations) + +**Problem**: Slow force simulation +**Solution**: Adaptive sampling, LOD techniques + +### Browser Compatibility +**Problem**: Old browser issues +**Solution**: Target modern browsers, provide fallbacks + +## Resources + +### Documentation +- `SDG_NETWORK_GUIDE.md` - Complete usage guide +- `specs/sdg_network_progressive.md` - Specification +- `specs/sdg_network_url_strategy.json` - Learning URLs +- `CLAUDE.md` - Project overview + +### Reference +- `ai_docs/sdg-network-modelling/` - Original baseline +- Observable HQ - D3.js examples +- GitHub public-apis - API directory + +### Commands +- `.claude/commands/infinite-web.md` - Implementation +- `/project:infinite-web` - Run command + +## Conclusion + +The new SDG Network Visualization system transforms a static, single-API visualization into a **progressive, web-learning-enhanced, multi-API discovery system** that: + +1. **Discovers** new data sources automatically via web search +2. **Learns** advanced techniques from curated web resources +3. **Integrates** multiple APIs into rich network visualizations +4. **Enhances** progressively across iterations +5. **Documents** sources and learnings clearly + +This system leverages the web-enhanced infinite loop pattern to create increasingly sophisticated SDG network visualizations that provide genuine value for sustainable development insights and data exploration. + +**The system is ready to run. Start with:** +```bash +claude +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 +``` diff --git a/SDG_NETWORK_GUIDE.md b/SDG_NETWORK_GUIDE.md new file mode 100644 index 0000000..8368001 --- /dev/null +++ b/SDG_NETWORK_GUIDE.md @@ -0,0 +1,404 @@ +# SDG Network Visualization - Web-Enhanced Infinite Loop Guide + +## Overview + +This guide explains how to use the web-enhanced infinite agentic loop to generate progressively sophisticated SDG (Sustainable Development Goals) network visualizations that integrate multiple open data APIs. + +## What Gets Generated + +Each iteration produces a **self-contained HTML file** with: +- **Force-directed network graph** using D3.js +- **Real data** from open APIs (environmental, scientific, SDG-related) +- **Progressive enhancements** in visualization quality and interactivity +- **Web-sourced techniques** applied and documented +- **API discovery** through automated web searches + +## Current Baseline + +The existing SDG network visualization (`ai_docs/sdg-network-modelling/`) provides: +- Basic D3.js v3 force-directed graph +- UN SDG API integration +- Simple node/edge rendering +- Basic hover tooltips +- Geographic and SDG goal relationships + +## Progressive Enhancement Strategy + +### **Phase 1: Foundation (Iterations 1-5)** +**Visual Enhancements:** +- Node size based on data magnitude +- Color coding by categories (using D3 color scales) +- Smart label positioning +- Rich hover tooltips with formatted data +- Edge width based on relationship strength +- Directional arrows and edge patterns + +**Interactivity:** +- Drag nodes to reposition +- Zoom and pan controls +- Click to highlight connected nodes +- Basic filtering by node type +- Legend for color coding + +**Data:** +- Single API per iteration +- Examples: UN SDG API, World Bank, OpenAQ, USGS Water + +### **Phase 2: Intermediate (Iterations 6-12)** +**Visual Enhancements:** +- Multi-property color encoding (fill color + border color) +- Icon overlays on nodes +- Curved edges for better visibility +- Node clustering by community +- Edge bundling for cleaner layouts +- Custom node shapes for different entity types + +**Interactivity:** +- Search functionality to find nodes +- Multi-select for comparison +- Time-based animation +- Expandable/collapsible groups +- Side panel with statistics +- Export visualization and data + +**Data:** +- Combine 2-3 API sources +- Automatic data merging +- Examples: NASA + NOAA + EPA, WHO + World Bank + UN + +### **Phase 3: Advanced (Iterations 13-20)** +**Visual Enhancements:** +- Force-based clustering algorithms +- Community detection visualization +- Centrality measures displayed +- Node importance ranking +- Hierarchical edge bundling +- Animated state transitions + +**Interactivity:** +- Real-time data updates +- Graph layout algorithm switching +- Advanced filtering (boolean logic) +- Shortest path finding +- Neighborhood exploration +- Graph metric dashboards + +**Data:** +- 4+ API sources integrated +- Automated API discovery via Swagger search +- Cross-API entity resolution +- Streaming data integration + +### **Phase 4: Expert (Iterations 21+)** +**Visual Enhancements:** +- ML-based optimal node positioning +- Anomaly detection highlighting +- Predictive attributes visualization +- 3D network projections +- VR/AR capabilities + +**Interactivity:** +- Natural language graph queries +- AI-powered insights +- Automatic pattern detection +- Graph comparison tools +- Collaborative features + +**Data:** +- Federated queries across APIs +- Knowledge graph construction +- Semantic web integration +- Blockchain data sources + +## API Discovery Process + +Each iteration automatically discovers new data sources: + +### **Target Domains:** +1. **Environmental**: Air quality, water quality, climate +2. **Scientific**: Biodiversity, satellites, research datasets +3. **Atmospheric**: Weather, air monitoring +4. **Ecological**: Forests, species, conservation +5. **Ocean**: Marine life, coastal data +6. **Geophysical**: Earthquakes, volcanoes +7. **Health**: Disease, mortality, healthcare +8. **SDG**: Development indicators + +### **Discovery Method:** +1. Web search: `"[domain] API swagger site:github.com"` +2. Parse Swagger/OpenAPI specifications +3. Evaluate API quality and relevance +4. Map data to network structure (nodes/edges) +5. Integrate and visualize + +### **API Examples:** +- **UN SDG API**: https://unstats.un.org/SDGAPI/v1/sdg/ +- **OpenAQ**: Air quality data worldwide +- **USGS Water Services**: Water quality and flow +- **Global Forest Watch**: Deforestation and forest data +- **NASA Earthdata**: Satellite and earth science +- **NOAA**: Weather, ocean, climate data +- **GBIF**: Biodiversity occurrence data +- **World Bank**: Development indicators + +## Web Learning Integration + +Each iteration learns from curated web resources: + +### **Foundation Topics:** +- D3.js fundamentals and data binding +- Basic force-directed graphs +- Color scales and node styling +- Interactive tooltips and zoom +- Canvas rendering for performance + +### **Intermediate Topics:** +- Force-based clustering +- Custom force implementations +- Advanced color schemes +- Animated transitions +- Modular component architecture + +### **Advanced Topics:** +- Edge bundling algorithms +- 3D projections and rotations +- Adaptive rendering for large graphs +- Real-time streaming data +- Multi-view interfaces + +### **Expert Topics:** +- Graph theory algorithms (centrality, shortest path) +- Community detection +- Semantic zoom +- Export capabilities +- ML-enhanced layouts + +## Usage Examples + +### **Single Iteration with Default Settings:** +```bash +claude +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 +``` + +This generates `sdg_viz/sdg_viz_1.html` with: +- UN SDG API baseline +- Basic force graph from D3.js documentation +- Foundation-level enhancements + +### **Small Batch (5 iterations):** +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 +``` + +Generates 5 files with: +- Different API sources (UN SDG, World Bank, OpenAQ, etc.) +- Progressive techniques from foundation URLs +- Each builds upon insights from previous iterations + +### **Medium Batch with Custom URL Strategy:** +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz 12 specs/sdg_network_url_strategy.json +``` + +Generates 12 iterations: +- Iterations 1-5: Foundation techniques +- Iterations 6-12: Intermediate techniques +- Multiple APIs combined +- Advanced interactivity + +### **Infinite Mode (Continuous until context limit):** +```bash +> /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite specs/sdg_network_url_strategy.json +``` + +Runs continuously with: +- Progressive difficulty waves +- Automatic API discovery via web search +- Increasingly sophisticated visualizations +- Stops gracefully at context limit + +## Expected Output Structure + +Each `sdg_viz_[N].html` file contains: + +```html + + + + SDG Network Visualization [N] - [Data Theme] + + + +
+ +
/* Interactive controls */
+
/* Data details */
+
/* Visual legend */
+
+ + + + + + + +``` + +## Key Improvements Over Baseline + +### **Data Integration:** +- ❌ **Old**: Single hardcoded API (UN SDG only) +- ✅ **New**: Automatic API discovery, multiple sources combined + +### **Visualization:** +- ❌ **Old**: Basic circles and lines, single color scheme +- ✅ **New**: Rich node/edge properties, dynamic sizing, multi-attribute encoding + +### **Interactivity:** +- ❌ **Old**: Simple hover tooltip +- ✅ **New**: Search, filter, zoom, click actions, side panels, exports + +### **Learning:** +- ❌ **Old**: Static implementation +- ✅ **New**: Each iteration learns from web and improves + +### **Technology:** +- ❌ **Old**: D3.js v3 (outdated) +- ✅ **New**: D3.js v7 (modern), latest techniques + +## Quality Metrics + +Each iteration is evaluated on: + +### **Data Richness:** +- Number of API sources (1 → 5+) +- Total nodes/edges +- Data update frequency + +### **Visual Quality:** +- Color coding sophistication +- Label readability +- Animation smoothness (60fps) +- Layout clarity + +### **Interactivity:** +- Number of features (hover → search → filter → AI) +- Response time +- Information density + +### **Technical Innovation:** +- New techniques applied +- Performance improvements +- Code quality + +## Advanced Use Cases + +### **Compare Algorithms:** +Generate iterations with different force algorithms: +```bash +# Each iteration learns a different layout algorithm +> /project:infinite-web specs/sdg_network_progressive.md algorithm_comparison 8 +``` + +### **Domain-Specific Networks:** +Focus on specific data domains by modifying the spec: +```bash +# Ocean/marine data network +> /project:infinite-web specs/ocean_network_progressive.md ocean_viz 10 + +# Climate change network +> /project:infinite-web specs/climate_network_progressive.md climate_viz 15 +``` + +### **Real-Time Dashboards:** +Advanced iterations include streaming data: +```bash +# Generate real-time monitoring dashboards +> /project:infinite-web specs/sdg_network_progressive.md realtime_sdg 20 +``` + +## Troubleshooting + +### **API Access Issues:** +- Some APIs require keys → iterations handle gracefully with fallbacks +- CORS errors → iterations use proxy or alternatives +- Rate limits → iterations implement caching + +### **Performance:** +- Large graphs → iterations use Canvas or WebGL +- Slow rendering → iterations implement LOD and culling +- Memory issues → iterations use virtualization + +### **Browser Compatibility:** +- All iterations target modern browsers (Chrome, Firefox, Safari, Edge) +- Fallbacks for older browsers where reasonable +- Progressive enhancement ensures basic functionality + +## Next Steps + +1. **Run your first iteration:** + ```bash + claude + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 1 + ``` + +2. **Review the output:** + - Open `sdg_viz/sdg_viz_1.html` in browser + - Check data sources and web learning attribution + - Test interactivity + +3. **Generate a batch:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz 5 + ``` + +4. **Compare iterations:** + - See progressive improvements + - Note new techniques applied + - Observe API diversity + +5. **Go infinite:** + ```bash + > /project:infinite-web specs/sdg_network_progressive.md sdg_viz infinite + ``` + +## Resources + +### **Specifications:** +- `specs/sdg_network_progressive.md` - Main visualization spec +- `specs/sdg_network_url_strategy.json` - Progressive learning URLs + +### **Commands:** +- `.claude/commands/infinite-web.md` - Web-enhanced loop implementation + +### **Baseline Reference:** +- `ai_docs/sdg-network-modelling/` - Original SDG network project + +### **Generated Output:** +- `sdg_viz/` - Output directory for iterations (create via command) + +## Contributing + +To extend this system: + +1. **Add new API sources** to `sdg_network_url_strategy.json` search templates +2. **Add learning resources** as new URLs in the strategy file +3. **Create domain-specific specs** by copying and modifying the base spec +4. **Share successful iterations** as examples + +## Credits + +- **Original SDG Network**: First Green Bank Network visualization +- **D3.js**: Michael Bostock and contributors +- **Observable**: Interactive notebook platform +- **Open APIs**: UN, World Bank, NOAA, USGS, NASA, and many others +- **Claude Code**: Infinite agentic loop implementation diff --git a/sdg_viz/sdg_viz_1.html b/sdg_viz/sdg_viz_1.html new file mode 100644 index 0000000..d51e8fe --- /dev/null +++ b/sdg_viz/sdg_viz_1.html @@ -0,0 +1,431 @@ + + + + + + SDG Network Visualization - Force-Directed Graph + + + + +
+
Loading UN SDG Data...
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/sdg_viz/sdg_viz_2.html b/sdg_viz/sdg_viz_2.html new file mode 100644 index 0000000..b70cf79 --- /dev/null +++ b/sdg_viz/sdg_viz_2.html @@ -0,0 +1,665 @@ + + + + + + SDG Network Viz 2: Environmental Indicators Network + + + + +
+
+

Environmental Indicators Network

+

Climate & Environment Data from World Bank Open Data API

+
+ +
+ + + + + + + +
+ +
+
+ + Loading environmental data from World Bank API... +
+
+ +
+

Legend

+
+

Node Colors (Indicator Categories)

+
+
+
+

Node Sizes (Data Magnitude)

+
+
+
+ Low values +
+
+
+ Medium values +
+
+
+ High values +
+
+
+
+

Edge Width (Correlation Strength)

+
+
+ + Weak correlation +
+
+ + Moderate correlation +
+
+ + Strong correlation +
+
+
+
+ +
+

Data Source & Learning Documentation

+

API Source: World Bank Open Data API (https://api.worldbank.org/v2/)

+

Indicators: CO2 Emissions (EN.ATM.CO2E.PC), Forest Area (AG.LND.FRST.ZS), Renewable Energy (EG.FEC.RNEW.ZS), Access to Electricity (EG.ELC.ACCS.ZS), PM2.5 Air Pollution (EN.ATM.PM25.MC.M3)

+

Web Learning Source: https://d3-graph-gallery.com/network.html

+

Techniques Applied:

+ +

Network Structure: Countries as nodes, environmental indicator correlations as edges. Nodes sized by normalized indicator values, colored by indicator category, connected when indicators show regional correlations.

+
+
+ +
+ + + + diff --git a/sdg_viz/sdg_viz_3.html b/sdg_viz/sdg_viz_3.html new file mode 100644 index 0000000..9459130 --- /dev/null +++ b/sdg_viz/sdg_viz_3.html @@ -0,0 +1,1014 @@ + + + + + + SDG Network Viz 3 - Global Biodiversity Interaction Network + + + + +
+
+

Interactive Controls

+ +
+

Search Species

+ + +
+ +
+

Filter by Taxonomic Rank

+
+
+ +
+

Actions

+ + + +
+ +
+

Node Details

+

Click a node to see details

+
+
+ +
+
+ + Loading biodiversity data from GBIF... +
+ +
+
+
+
+
+ + + + + + diff --git a/sdg_viz/sdg_viz_4.html b/sdg_viz/sdg_viz_4.html new file mode 100644 index 0000000..d3aa3e1 --- /dev/null +++ b/sdg_viz/sdg_viz_4.html @@ -0,0 +1,720 @@ + + + + + + SDG Network - Air Quality Monitoring (Performance Optimized) + + + + +
+
+

🌍 Global Air Quality Network

+

SDG 11: Sustainable Cities & Communities - Performance Optimized Edition

+
+ +
+
+
+ Initializing... +
+
+ Load time: -- +
+
+ +
+ + + +
+ +
+ +
+
Air Quality Categories
+
+
+
+ +
+
+
+
Cities Monitored
+
--
+
+
+
Network Connections
+
--
+
+
+
Avg PM2.5 (μg/m³)
+
--
+
+
+
Data Source
+
--
+
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/sdg_viz/sdg_viz_5.html b/sdg_viz/sdg_viz_5.html new file mode 100644 index 0000000..fbcaf99 --- /dev/null +++ b/sdg_viz/sdg_viz_5.html @@ -0,0 +1,739 @@ + + + + + + SDG Network Viz 5 - Advanced Color Encodings & Visual Hierarchy + + + + +
+
+

SDG Network Visualization - Advanced Color Encodings

+

Multi-Dimensional Color Strategy with Visual Hierarchy

+
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+ +
+ +
+ +
+
+
+ 🎨 + SDG Goal Categories (Click to Filter) +
+
+
+ +
+
+ 📊 + Impact Magnitude (Border Color & Size) +
+
+
+
+
+ Low Impact + Medium Impact + High Impact +
+
+
+
+ +
+
+ 💫 + Data Quality (Opacity) +
+
+
+
+
+ Low Quality (40%) +
+
+
+ Medium Quality (70%) +
+
+
+ High Quality (100%) +
+
+
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/sdg_viz/sdg_viz_6.html b/sdg_viz/sdg_viz_6.html new file mode 100644 index 0000000..214ee36 --- /dev/null +++ b/sdg_viz/sdg_viz_6.html @@ -0,0 +1,974 @@ + + + + + + SDG Network Viz 6 - ETL Pipeline with Embedded Multi-Year Data + + + + +
+ + +
+ +
+
+
+
+ + + + + + diff --git a/sdg_viz/sdg_viz_7.html b/sdg_viz/sdg_viz_7.html new file mode 100644 index 0000000..8e519cd --- /dev/null +++ b/sdg_viz/sdg_viz_7.html @@ -0,0 +1,904 @@ + + + + + + SDG Network - Exploratory Analysis: Brushing & Linking + + + + +
+

SDG Climate Network - Exploratory Analysis

+
Brushing & Linking for Multi-View Coordination
+
+ +
+ + +
+ +
+
+
+ Force Network View + 0 selected +
+ +
+ +
+
+ Connection Degree Distribution + 0 selected +
+ +
+ +
+
+ Impact vs Connectivity + 0 selected +
+ +
+
+ +
+ +
+ Iteration 7: Exploratory Analysis with Brushing & Linking
+ This visualization demonstrates advanced exploratory data analysis using coordinated multiple views. + Rectangle brush on the network to select nodes, and observe synchronized highlighting across all views. + Features d3.brush() for selection, coordinated view updates, and multi-select capability (Shift+Click).
+ Web Learning: Techniques from D3 Brushable Scatterplot - + Applied brush event handling, selection propagation via dispatch, coordinated filtering across views, and synchronized visual feedback.
+ Data: Embedded climate action network (50 nodes, 85 connections) - Instant load, offline-ready +
+ + + + diff --git a/sdg_viz/sdg_viz_8.html b/sdg_viz/sdg_viz_8.html new file mode 100644 index 0000000..93251e2 --- /dev/null +++ b/sdg_viz/sdg_viz_8.html @@ -0,0 +1,970 @@ + + + + + + SDG Network Viz 8 - Hierarchical SDG Taxonomy with Multi-Layout Transitions + + + + +
+
+

Hierarchical Controls

+ +
+

Layout Mode

+ + + + +
+ +
+

Hierarchy Actions

+ + + + +
+ +
+

Node Information

+

Click a node to view details
Click again to expand/collapse children

+
+
+ +
+ +
+
+
+
+
+ + + + + + diff --git a/sdg_viz/sdg_viz_9.html b/sdg_viz/sdg_viz_9.html new file mode 100644 index 0000000..f5a5f1e --- /dev/null +++ b/sdg_viz/sdg_viz_9.html @@ -0,0 +1,916 @@ + + + + + + SDG Network Visualization 9 - High Performance Canvas Rendering + + + + +
+ + +
+

⚙️ Performance Controls

+ +
+ +
+ + +
+
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+

📊 Performance Metrics

+
+ FPS: + 60 +
+
+ Visible Nodes: + 0 +
+
+ Total Nodes: + 0 +
+
+ Render Time: + 0ms +
+
+ Simulation: + Running +
+
+ Quadtree Depth: + 0 +
+
+ +
+
Loading nodes progressively...
+
+
+
+
+ +
+

SDG Categories

+
+ +
+
+
+
+ + +
+ + + + \ No newline at end of file diff --git a/specs/sdg_network_progressive.md b/specs/sdg_network_progressive.md new file mode 100644 index 0000000..8fb7b79 --- /dev/null +++ b/specs/sdg_network_progressive.md @@ -0,0 +1,446 @@ +# Progressive SDG Network Visualization Specification + +## Core Challenge +Create **progressively sophisticated SDG network visualizations** that integrate multiple open data APIs to display environmental, scientific, and sustainable development data as interactive force-directed graphs. Each iteration should discover new data sources and enhance visual/interactive capabilities. + +## Output Requirements + +**File Naming**: `sdg_viz_[iteration_number].html` + +**Content Structure**: Self-contained HTML file with embedded D3.js visualization +```html + + + + + + SDG Network Visualization [Iteration Number] - [Data Source Theme] + + + +
+ + + + +
+ +
+ + +
+ +
+ + +
+ +
+
+ + + + + + + + + + +``` + +## Progressive Enhancement Dimensions + +### **Phase 1: Foundation (Iterations 1-5)** +**Focus**: Basic network visualization with single API integration + +**Node Enhancements:** +- Size based on data properties (e.g., population, magnitude) +- Color coding by category/type (using color scales) +- Clear labels with smart positioning +- Hover tooltips with basic information + +**Edge Enhancements:** +- Width based on relationship strength +- Color coding for relationship types +- Straight lines with arrows for directionality +- Dashed/solid patterns for different relationship classes + +**Interactivity:** +- Drag nodes to reposition +- Zoom and pan controls +- Click to highlight connected nodes +- Basic filtering by node type + +**Data Integration:** +- Single API source per iteration +- Simple JSON data transformation +- Basic node-edge relationship mapping +- Manual API endpoint definition + +### **Phase 2: Intermediate (Iterations 6-12)** +**Focus**: Multi-source integration and enhanced interactivity + +**Node Enhancements:** +- Dynamic sizing with min/max constraints +- Multi-property color encoding (fill + border) +- Icon/symbol overlays for node types +- Rich tooltips with formatted data tables +- Node clustering and grouping +- Custom shapes for different entity types + +**Edge Enhancements:** +- Curved edges for better visibility +- Dynamic edge strength from data properties +- Animated flow along edges +- Edge bundling for cleaner layouts +- Multiple edge types with different styles + +**Interactivity:** +- Search functionality to find nodes +- Multi-select for comparison +- Time-based filtering/animation +- Expandable/collapsible node groups +- Side panel with detailed statistics +- Export data/visualization options + +**Data Integration:** +- Combine 2-3 API sources +- Automatic data merging by key fields +- API discovery via Swagger/OpenAPI specs +- Error handling for API failures +- Caching for performance + +### **Phase 3: Advanced (Iterations 13-20)** +**Focus**: Sophisticated algorithms and real-time capabilities + +**Node Enhancements:** +- Force-based clustering algorithms +- Community detection visualization +- Centrality measures (betweenness, closeness) +- Node importance ranking +- Multi-level node hierarchies +- Animated node state transitions + +**Edge Enhancements:** +- Weighted force simulation +- Edge confidence/uncertainty visualization +- Temporal edge evolution +- Hierarchical edge bundling +- Edge crossing minimization + +**Interactivity:** +- Real-time data updates +- Graph layout algorithm selection +- Advanced filtering with boolean logic +- Shortest path highlighting +- Neighborhood exploration +- Graph metric dashboards +- Collaborative filtering + +**Data Integration:** +- 4+ API sources integrated +- Automated API endpoint discovery +- Data quality indicators +- Cross-API entity resolution +- Streaming data integration +- Webhook support for updates + +### **Phase 4: Expert (Iterations 21+)** +**Focus**: Novel techniques and AI-enhanced features + +**Node Enhancements:** +- ML-based node positioning +- Anomaly detection highlighting +- Predictive node attributes +- Semantic node embedding +- Dynamic node importance +- Multi-dimensional projection + +**Edge Enhancements:** +- Probabilistic edge rendering +- Predicted relationship suggestions +- Edge importance scoring +- Dynamic edge weight learning +- Causal relationship inference + +**Interactivity:** +- Natural language queries +- AI-powered graph insights +- Automatic pattern detection +- Interactive graph queries (Cypher-like) +- Graph comparison tools +- VR/AR integration capabilities + +**Data Integration:** +- Federated query across APIs +- Automatic schema alignment +- Knowledge graph construction +- Semantic web integration (RDF, SPARQL) +- Real-time distributed data +- Blockchain data sources + +## API Discovery Strategy + +### **Target Data Domains** +Each iteration should discover and integrate APIs from these domains: + +**Environmental Data:** +- Air quality (OpenAQ, EPA APIs) +- Water quality (USGS, EPA Water) +- Climate data (NOAA, NASA) +- Weather (OpenWeatherMap, Weather.gov) +- Ocean data (NOAA Ocean, Marine APIs) + +**Scientific Data:** +- Biodiversity (GBIF, iNaturalist) +- Forest data (Global Forest Watch) +- Soil data (SoilGrids, ISRIC) +- Satellite imagery (NASA Earth, Sentinel) +- Seismic data (USGS Earthquake) + +**Sustainable Development:** +- UN SDG API (existing baseline) +- World Bank Open Data +- OECD Statistics +- WHO Global Health +- FAO Agricultural Data + +**Discovery Process:** +1. **Web Search for APIs**: Use WebSearch to find: `"[domain] API swagger site:github.com"` +2. **Swagger/OpenAPI Analysis**: Parse API specs to find endpoints +3. **Data Quality Check**: Verify API has relevant network-ready data +4. **Integration Planning**: Map API data to nodes and edges +5. **Documentation**: Record API details in visualization + +### **API Selection Criteria** +- **Open Access**: Free or freemium tier available +- **REST/JSON**: Easy integration with JavaScript +- **Documentation**: Well-documented endpoints +- **Reliability**: Active maintenance and uptime +- **Data Richness**: Complex enough for network visualization +- **SDG Relevance**: Connects to sustainability goals + +## Network Data Model + +### **Node Schema** +```javascript +{ + id: "unique_identifier", + name: "Display Name", + type: "category", // e.g., "country", "indicator", "species" + group: "group_name", // for color coding + size: numeric_value, // for visual sizing + properties: { + // Domain-specific attributes + value: number, + unit: "string", + timestamp: "ISO_date", + source_api: "api_name", + // ... additional properties + }, + x: number, // force layout position + y: number +} +``` + +### **Edge Schema** +```javascript +{ + source: "node_id", + target: "node_id", + type: "relationship_type", + strength: numeric_value, // for force simulation + weight: numeric_value, // for visual width + properties: { + correlation: number, + confidence: number, + source_api: "api_name", + // ... additional properties + } +} +``` + +## Web Learning Integration + +### **Each Iteration Must:** +1. **Fetch Assigned URL**: Use WebFetch to retrieve documentation/tutorial +2. **Extract Techniques**: Identify 1-3 specific techniques from the source +3. **Apply Learning**: Implement the technique in the visualization +4. **Document Source**: Clearly attribute the web source and what was learned +5. **Demonstrate Improvement**: Show measurable enhancement from previous iteration + +### **Web Source Categories** + +**D3.js Techniques:** +- Observable HQ notebooks (d3.js examples) +- D3.js official documentation +- Mike Bostock's blocks +- D3 Graph Gallery +- Interactive graph tutorials + +**Force-Directed Graphs:** +- Force simulation parameters +- Collision detection +- Link distance functions +- Clustering algorithms +- Layout optimization + +**Data Integration:** +- REST API patterns +- CORS handling +- Async data loading +- Data transformation pipelines +- Error handling strategies + +**Visual Design:** +- Color theory for networks +- Graph readability principles +- Interactive UX patterns +- Animation best practices +- Accessibility in visualizations + +## Quality Standards + +### **Functional Requirements** +- ✅ Fetches data from at least one open API +- ✅ Renders as force-directed graph with D3.js +- ✅ Nodes and edges have meaningful properties from data +- ✅ Interactive (hover, click, drag minimum) +- ✅ Responsive design works on desktop and mobile +- ✅ Clear data source attribution +- ✅ Error handling for API failures + +### **Visual Excellence** +- ✅ Professional color scheme with good contrast +- ✅ Readable labels and legends +- ✅ Smooth animations (60fps) +- ✅ Clear visual hierarchy +- ✅ Consistent styling throughout +- ✅ Loading states for async operations + +### **Technical Quality** +- ✅ Self-contained HTML file (or minimal external deps) +- ✅ Clean, commented JavaScript code +- ✅ Efficient force simulation (no lag) +- ✅ Proper error handling and fallbacks +- ✅ Browser compatibility (modern browsers) +- ✅ No console errors + +### **Progressive Learning** +- ✅ Demonstrates new technique from web source +- ✅ Builds upon previous iterations where appropriate +- ✅ Documents what was learned and how applied +- ✅ Shows measurable improvement +- ✅ Explores new API sources + +## Iteration Evolution Strategy + +### **Starting Point (Iteration 1)** +- Use UN SDG API (baseline from existing project) +- Basic force-directed graph with D3.js v7 +- Simple node/edge styling +- Basic hover tooltips +- Learn from: D3.js force simulation documentation + +### **Early Iterations (2-5)** +- Add one new API per iteration +- Focus on core graph enhancements +- Build interactive controls +- Learn: Color scales, tooltips, filtering + +### **Middle Iterations (6-12)** +- Combine multiple APIs +- Advanced interactivity +- Custom layouts and algorithms +- Learn: Graph algorithms, data merging, UX patterns + +### **Advanced Iterations (13-20)** +- Real-time data integration +- Complex algorithms (community detection, centrality) +- Rich dashboards +- Learn: Advanced D3, graph theory, streaming data + +### **Expert Iterations (21+)** +- AI/ML integration +- Novel visualization techniques +- Cross-platform capabilities +- Learn: Cutting-edge graph visualization research + +## Success Metrics + +Each iteration should improve on at least 3 of these metrics: + +**Data Richness:** +- Number of API sources integrated +- Total nodes in network +- Total edges in network +- Data update frequency + +**Visual Quality:** +- Color coding sophistication +- Label readability score +- Animation smoothness +- Layout clarity + +**Interactivity:** +- Number of interactive features +- Response time to user actions +- Filter/search capabilities +- Information density + +**Technical Innovation:** +- New techniques applied +- Performance improvements +- Code quality metrics +- Browser compatibility + +## Ultra-Thinking Directive + +Before each iteration, deeply consider: + +**API Discovery:** +- What domain provides the most value for SDG insights? +- Which APIs have network-ready data structures? +- How can multiple APIs be meaningfully combined? +- What data relationships would be most revealing? + +**Visualization Strategy:** +- What graph layout best reveals the data patterns? +- How can node/edge properties enhance understanding? +- What interactivity would provide the most insight? +- How does this build on previous iterations? + +**Web Learning Application:** +- What specific technique from the web source applies here? +- How can this technique improve the visualization? +- What adaptations are needed for our use case? +- How does this advance the state of the art? + +**User Value:** +- What insights does this visualization reveal? +- How intuitive is the interaction model? +- What questions can users answer with this tool? +- How does this serve SDG analysis and understanding? + +**Generate visualizations that:** +- **Discover New Data**: Find and integrate novel API sources +- **Apply Web Knowledge**: Demonstrate clear learning from web research +- **Enhance Progressively**: Each iteration measurably better than the last +- **Serve SDGs**: Provide genuine value for sustainable development insights +- **Inspire Exploration**: Create compelling, informative, interactive experiences diff --git a/specs/sdg_network_url_strategy.json b/specs/sdg_network_url_strategy.json new file mode 100644 index 0000000..1b633aa --- /dev/null +++ b/specs/sdg_network_url_strategy.json @@ -0,0 +1,278 @@ +{ + "description": "Progressive web learning strategy for SDG network visualizations with API integration", + "categories": { + "foundation": { + "description": "Basic D3.js force-directed graphs and API integration (Iterations 1-5)", + "urls": [ + { + "url": "https://d3js.org/what-is-d3", + "topic": "D3.js fundamentals and data binding", + "techniques": ["data binding", "selections", "enter/update/exit pattern"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph", + "topic": "Basic force-directed graph implementation", + "techniques": ["force simulation", "node positioning", "link rendering"] + }, + { + "url": "https://d3-graph-gallery.com/network.html", + "topic": "Network visualization gallery and examples", + "techniques": ["color scales", "node sizing", "edge styling"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph-canvas", + "topic": "Canvas-based force graphs for performance", + "techniques": ["canvas rendering", "performance optimization", "event handling"] + }, + { + "url": "https://observablehq.com/@d3/temporal-force-directed-graph", + "topic": "Adding interactivity to force graphs", + "techniques": ["drag behavior", "zoom", "tooltip interactions"] + } + ] + }, + "intermediate": { + "description": "Advanced D3 techniques and multi-API integration (Iterations 6-12)", + "urls": [ + { + "url": "https://observablehq.com/@d3/sticky-force-layout", + "topic": "Sticky force layout for better control", + "techniques": ["fixed positions", "sticky drag", "layout constraints"] + }, + { + "url": "https://observablehq.com/@d3/clustered-force-layout", + "topic": "Force-based clustering", + "techniques": ["cluster forces", "group positioning", "collision detection"] + }, + { + "url": "https://observablehq.com/@d3/arc-diagram", + "topic": "Alternative network layouts", + "techniques": ["arc diagrams", "hierarchical layouts", "radial layouts"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-lattice", + "topic": "Custom force implementations", + "techniques": ["custom forces", "lattice constraints", "force composition"] + }, + { + "url": "https://observablehq.com/@d3/color-schemes", + "topic": "Advanced color schemes for networks", + "techniques": ["sequential scales", "diverging scales", "categorical colors"] + }, + { + "url": "https://observablehq.com/@d3/connected-particles", + "topic": "Animated network transitions", + "techniques": ["particle systems", "edge animation", "smooth transitions"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph-component", + "topic": "Modular graph components", + "techniques": ["component architecture", "reusable patterns", "state management"] + } + ] + }, + "advanced": { + "description": "Complex algorithms and real-time data (Iterations 13-20)", + "urls": [ + { + "url": "https://observablehq.com/@d3/hierarchical-edge-bundling", + "topic": "Edge bundling for clarity", + "techniques": ["hierarchical bundling", "edge aggregation", "path smoothing"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-tree", + "topic": "Force-directed trees and hierarchies", + "techniques": ["tree forces", "hierarchy layout", "parent-child links"] + }, + { + "url": "https://observablehq.com/@mbostock/adaptive-sampling", + "topic": "Adaptive rendering for large graphs", + "techniques": ["level of detail", "culling", "adaptive sampling"] + }, + { + "url": "https://observablehq.com/@d3/versor-dragging", + "topic": "3D network rotations", + "techniques": ["quaternion rotation", "3D projection", "spherical layout"] + }, + { + "url": "https://observablehq.com/@d3/voronoi-labels", + "topic": "Smart label placement", + "techniques": ["voronoi diagrams", "collision-free labels", "dynamic positioning"] + }, + { + "url": "https://observablehq.com/@d3/zoom-to-bounding-box", + "topic": "Advanced zoom and focus", + "techniques": ["semantic zoom", "focus+context", "smooth camera transitions"] + }, + { + "url": "https://observablehq.com/@mbostock/the-wealth-health-of-nations", + "topic": "Real-time data integration", + "techniques": ["data streaming", "time series", "dynamic updates"] + }, + { + "url": "https://observablehq.com/@d3/graph-layout-force-directed", + "topic": "Graph layout algorithms comparison", + "techniques": ["layout selection", "parameter tuning", "performance trade-offs"] + } + ] + }, + "expert": { + "description": "Cutting-edge techniques and AI integration (Iterations 21+)", + "urls": [ + { + "url": "https://observablehq.com/@d3/force-directed-graph-with-minimap", + "topic": "Multi-view graph interfaces", + "techniques": ["minimap", "multiple synchronized views", "overview+detail"] + }, + { + "url": "https://observablehq.com/@mbostock/phase-portrait", + "topic": "Dynamic system visualization", + "techniques": ["phase space", "attractor visualization", "vector fields"] + }, + { + "url": "https://observablehq.com/@fil/occlusion-hello-world", + "topic": "3D occlusion and depth", + "techniques": ["depth sorting", "occlusion culling", "3D rendering"] + }, + { + "url": "https://observablehq.com/@d3/graph", + "topic": "Graph theory algorithms", + "techniques": ["shortest path", "centrality measures", "community detection"] + }, + { + "url": "https://observablehq.com/@mbostock/saving-svg", + "topic": "Export and sharing capabilities", + "techniques": ["SVG export", "image generation", "data export"] + } + ] + }, + "api_discovery": { + "description": "Web search templates for finding open APIs", + "search_templates": [ + { + "query": "environmental data API swagger site:github.com", + "domain": "Environmental APIs", + "expected_sources": ["air quality", "water quality", "climate data"] + }, + { + "query": "scientific data REST API openapi", + "domain": "Scientific Data", + "expected_sources": ["biodiversity", "satellite", "research datasets"] + }, + { + "query": "sustainable development goals API JSON", + "domain": "SDG Data", + "expected_sources": ["UN data", "World Bank", "development indicators"] + }, + { + "query": "atmospheric weather data API free", + "domain": "Atmospheric Data", + "expected_sources": ["weather", "air quality", "climate"] + }, + { + "query": "water quality watershed API REST", + "domain": "Water Data", + "expected_sources": ["USGS", "EPA", "water monitoring"] + }, + { + "query": "forest biodiversity data API open source", + "domain": "Ecological Data", + "expected_sources": ["Global Forest Watch", "species data", "conservation"] + }, + { + "query": "NOAA ocean marine data API", + "domain": "Ocean Data", + "expected_sources": ["ocean temperature", "marine life", "coastal data"] + }, + { + "query": "NASA earth observation API", + "domain": "Satellite Data", + "expected_sources": ["satellite imagery", "earth science", "remote sensing"] + }, + { + "query": "seismic earthquake API USGS", + "domain": "Geophysical Data", + "expected_sources": ["earthquake", "volcano", "geological"] + }, + { + "query": "WHO global health statistics API", + "domain": "Health Data", + "expected_sources": ["disease", "healthcare", "mortality"] + } + ], + "api_evaluation_criteria": [ + "Has Swagger/OpenAPI specification", + "Provides JSON/REST endpoints", + "Free tier or open access available", + "Good documentation", + "Active maintenance (updated recently)", + "Network-ready data (entities with relationships)", + "Relevant to SDGs or sustainability" + ] + }, + "data_integration": { + "description": "Resources for API integration and data merging", + "urls": [ + { + "url": "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch", + "topic": "Modern fetch API usage", + "techniques": ["async/await", "error handling", "CORS"] + }, + { + "url": "https://observablehq.com/@observablehq/introduction-to-data", + "topic": "Data loading and transformation", + "techniques": ["data parsing", "cleaning", "transformation pipelines"] + }, + { + "url": "https://observablehq.com/@d3/d3-fetch", + "topic": "D3 data fetching utilities", + "techniques": ["d3.json", "d3.csv", "data type inference"] + } + ] + }, + "graph_algorithms": { + "description": "Graph theory and network analysis resources", + "urls": [ + { + "url": "https://observablehq.com/@d3/disjoint-force-directed-graph", + "topic": "Disconnected graph handling", + "techniques": ["component detection", "separate force simulations"] + }, + { + "url": "https://observablehq.com/@d3/force-directed-graph?intent=fork", + "topic": "Customizable force parameters", + "techniques": ["force strength tuning", "link distance optimization"] + }, + { + "url": "https://observablehq.com/@mbostock/paths-in-a-force-directed-graph", + "topic": "Path finding and highlighting", + "techniques": ["shortest path", "path visualization", "connected components"] + } + ] + } + }, + "iteration_mapping": { + "1-5": "foundation", + "6-12": "intermediate", + "13-20": "advanced", + "21+": "expert" + }, + "fallback_strategy": { + "description": "When pre-defined URLs are exhausted, use dynamic web search", + "search_patterns": [ + "D3.js force directed graph [specific_technique]", + "network visualization [technique] observable", + "graph layout algorithm [specific_algorithm]", + "[api_domain] open API swagger github", + "force simulation [parameter] tuning d3" + ] + }, + "priming_urls": { + "description": "Initial URLs to fetch before starting any iteration for foundational knowledge", + "urls": [ + "https://d3js.org/what-is-d3", + "https://observablehq.com/@d3/force-directed-graph", + "https://github.com/public-apis/public-apis" + ] + } +}