Distributed Systems | Tags

<h2 id="distributed-systems-in-geode" class="position-relative d-flex align-items-center group"> Distributed Systems in Geode <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="distributed-systems-in-geode" aria-haspopup="dialog" aria-label="Share link: Distributed Systems in Geode"> Share link </button> </h2><div id="headingShareModal" class="heading-share-modal" role="dialog" aria-modal="true" aria-labelledby="headingShareTitle" hidden> <div class="hsm-dialog" role="document"> <div class="hsm-header"> <h2 id="headingShareTitle" class="h6 mb-0 fw-bold">Share this section</h2> <button type="button" class="hsm-close" aria-label="Close"> </button> </div> <div class="hsm-body"> <label for="headingShareInput" class="form-label small text-muted mb-1 text-uppercase fw-bold" style="font-size: 0.7rem; letter-spacing: 0.5px;">Permalink</label> <div class="input-group mb-4 hsm-url-group"> <input id="headingShareInput" type="text" class="form-control font-monospace" readonly aria-readonly="true" style="font-size: 0.85rem;" /> <button class="btn btn-primary hsm-copy" type="button" aria-label="Copy" title="Copy"> </button> </div> <div class="small fw-bold mb-2 text-muted text-uppercase" style="font-size: 0.7rem; letter-spacing: 0.5px;">Share via</div> <div class="hsm-share-grid"> <a id="share-twitter" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Twitter </a> <a id="share-linkedin" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> LinkedIn </a> <a id="share-facebook" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Facebook </a> </div> </div> </div> </div> <style> .heading-share-modal { position: fixed; inset: 0; display: flex; justify-content: center; align-items: center; background: rgba(0, 0, 0, 0.6); z-index: 1050; padding: 1rem; backdrop-filter: blur(4px); -webkit-backdrop-filter: blur(4px); } .heading-share-modal[hidden] { display: none !important; } .hsm-dialog { max-width: 420px; width: 100%; background: var(--bs-body-bg, #fff); color: var(--bs-body-color, #212529); border: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); border-radius: 1rem; box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.25); overflow: hidden; animation: hsm-fade-in 0.2s ease-out; } @keyframes hsm-fade-in { from { opacity: 0; transform: scale(0.95); } to { opacity: 1; transform: scale(1); } } [data-bs-theme="dark"] .hsm-dialog { background: #1e293b; border-color: rgba(255,255,255,0.1); color: #f8f9fa; } .hsm-header { display: flex; justify-content: space-between; align-items: center; padding: 1rem 1.5rem; border-bottom: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); background: rgba(0,0,0,0.02); } [data-bs-theme="dark"] .hsm-header { background: rgba(255,255,255,0.02); border-color: rgba(255,255,255,0.1); } .hsm-close { background: transparent; border: none; color: inherit; opacity: 0.5; padding: 0.25rem 0.5rem; border-radius: 0.25rem; font-size: 1.2rem; line-height: 1; transition: opacity 0.2s; } .hsm-close:hover { opacity: 1; } .hsm-body { padding: 1.5rem; } .hsm-url-group { display: flex !important; align-items: stretch; } .hsm-url-group .form-control { flex: 1; min-width: 0; margin: 0; background: var(--bs-secondary-bg, #f8f9fa); border-color: var(--bs-border-color, #dee2e6); border-top-right-radius: 0; border-bottom-right-radius: 0; height: 42px; } .hsm-url-group .btn { flex: 0 0 auto; margin: 0; margin-left: -1px; border-top-left-radius: 0; border-bottom-left-radius: 0; height: 42px; display: flex; align-items: center; justify-content: center; padding: 0 1.25rem; z-index: 2; } [data-bs-theme="dark"] .hsm-url-group .form-control { background: #0f172a; border-color: #334155; color: #e2e8f0; } .hsm-share-grid { display: flex; flex-direction: column; gap: 0.5rem; } .hsm-share-grid .btn { display: flex; align-items: center; justify-content: center; font-size: 0.9rem; padding: 0.6rem; border-color: var(--bs-border-color); width: 100%; } [data-bs-theme="dark"] .hsm-share-grid .btn { color: #e2e8f0; border-color: #475569; } [data-bs-theme="dark"] .hsm-share-grid .btn:hover { background: #334155; border-color: #cbd5e1; } </style> <script> (function(){ const modal = document.getElementById('headingShareModal'); if(!modal) return; const input = modal.querySelector('#headingShareInput'); const copyBtn = modal.querySelector('.hsm-copy'); const twitter = modal.querySelector('#share-twitter'); const linkedin = modal.querySelector('#share-linkedin'); const facebook = modal.querySelector('#share-facebook'); const closeBtn = modal.querySelector('.hsm-close'); let lastFocus=null; let trapBound=false; function buildUrl(id){ return window.location.origin + window.location.pathname + '#' + id; } function isOpen(){ return !modal.hasAttribute('hidden'); } function hydrate(id){ const url=buildUrl(id); input.value=url; const enc=encodeURIComponent(url); const text=encodeURIComponent(document.title); if(twitter) twitter.href=`https://twitter.com/intent/tweet?url=${enc}&text=${text}`; if(linkedin) linkedin.href=`https://www.linkedin.com/sharing/share-offsite/?url=${enc}`; if(facebook) facebook.href=`https://www.facebook.com/sharer/sharer.php?u=${enc}`; } function openModal(id){ lastFocus=document.activeElement; hydrate(id); if(!isOpen()){ modal.removeAttribute('hidden'); } requestAnimationFrame(()=>{ input.focus(); }); trapFocus(); } function closeModal(){ if(!isOpen()) return; modal.setAttribute('hidden',''); if(lastFocus && typeof lastFocus.focus==='function') lastFocus.focus(); } function copyCurrent(){ try{ navigator.clipboard.writeText(input.value).then(()=>feedback(true),()=>fallback()); } catch(e){ fallback(); } } function fallback(){ input.select(); try{ document.execCommand('copy'); feedback(true);}catch(e){ feedback(false);} } function feedback(ok){ if(!copyBtn) return; const icon=copyBtn.querySelector('i'); if(!icon) return; const prev=copyBtn.getAttribute('data-prev')||icon.className; if(!copyBtn.getAttribute('data-prev')) copyBtn.setAttribute('data-prev',prev); icon.className= ok ? 'fa-duotone fa-clipboard-check':'fa-duotone fa-circle-exclamation'; setTimeout(()=>{ icon.className=prev; },1800); } function handleShareClick(e){ e.preventDefault(); const btn=e.currentTarget; const id=btn.getAttribute('data-share-target'); if(id) openModal(id); } function bindShareButtons(){ document.querySelectorAll('.h-share').forEach(btn=>{ if(!btn.dataset.hShareBound){ btn.addEventListener('click', handleShareClick); btn.dataset.hShareBound='1'; } }); } bindShareButtons(); if(document.readyState==='loading'){ document.addEventListener('DOMContentLoaded', bindShareButtons); } else { requestAnimationFrame(bindShareButtons); } document.addEventListener('click', function(e){ const shareBtn=e.target.closest && e.target.closest('.h-share'); if(shareBtn && !shareBtn.dataset.hShareBound){ handleShareClick.call(shareBtn, e); } }, true); document.addEventListener('click', e=>{ if(e.target===modal) closeModal(); if(e.target.closest && e.target.closest('.hsm-close')){ e.preventDefault(); closeModal(); } if(copyBtn && (e.target===copyBtn || (e.target.closest && e.target.closest('.hsm-copy')))) { e.preventDefault(); copyCurrent(); } }); document.addEventListener('keydown', e=>{ if(e.key==='Escape' && isOpen()) closeModal(); }); function trapFocus(){ if(trapBound) return; trapBound=true; modal.addEventListener('keydown', f=>{ if(f.key==='Tab' && isOpen()){ const focusable=[...modal.querySelectorAll('a[href],button,input,textarea,select,[tabindex]:not([tabindex="-1"])')].filter(el=>!el.hasAttribute('disabled')); if(!focusable.length) return; const first=focusable[0]; const last=focusable[focusable.length-1]; if(f.shiftKey && document.activeElement===first){ f.preventDefault(); last.focus(); } else if(!f.shiftKey && document.activeElement===last){ f.preventDefault(); first.focus(); } } }); } if(closeBtn) closeBtn.addEventListener('click', e=>{ e.preventDefault(); closeModal(); }); })(); </script>Distributed systems enable Geode to scale beyond the limits of a single machine while maintaining consistency, availability, and fault tolerance. Geode implements a sophisticated distributed architecture designed specifically for graph workloads, where data locality and relationship traversal efficiency are paramount. This guide covers Geode’s distributed architecture, consensus mechanisms, cluster coordination, and strategies for building resilient, scalable graph database deployments. <h3 id="introduction-to-distributed-graph-databases" class="position-relative d-flex align-items-center group"> Introduction to Distributed Graph Databases <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="introduction-to-distributed-graph-databases" aria-haspopup="dialog" aria-label="Share link: Introduction to Distributed Graph Databases"> Share link </button> </h3>Traditional relational databases can be distributed by sharding rows across nodes, but graph databases face unique challenges: Relationship Locality: Graph traversals cross node boundaries, requiring efficient cross-shard communication Hotspot Prevention: Popular nodes (celebrities, viral content) can create load imbalances Consistency Across Edges: Relationships connecting nodes on different shards must remain consistent Variable Query Patterns: Graph queries follow unpredictable paths unlike tabular scans Geode addresses these challenges through intelligent data placement, distributed query planning, and optimized cross-shard communication. <h3 id="geode-cluster-architecture" class="position-relative d-flex align-items-center group"> Geode Cluster Architecture <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="geode-cluster-architecture" aria-haspopup="dialog" aria-label="Share link: Geode Cluster Architecture"> Share link </button> </h3> <h4 id="cluster-components" class="position-relative d-flex align-items-center group"> Cluster Components <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="cluster-components" aria-haspopup="dialog" aria-label="Share link: Cluster Components"> Share link </button> </h4>A Geode cluster consists of multiple node types working together: Leader Node: Coordinates cluster state and manages consensus <ul> <li>Handles schema changes and DDL operations</li> <li>Coordinates distributed transactions</li> <li>Manages cluster membership</li> <li>Performs global query planning</li> </ul> Data Nodes: Store and serve graph data <ul> <li>Host data partitions (shards)</li> <li>Execute local query operations</li> <li>Participate in consensus for writes</li> <li>Maintain local indexes</li> </ul> Query Coordinators: Route and aggregate queries <ul> <li>Parse and plan distributed queries</li> <li>Coordinate cross-shard operations</li> <li>Aggregate results from multiple nodes</li> <li>Handle client connections</li> </ul> <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml"># geode.toml - Cluster configuration [cluster] name = "production-cluster" mode = "distributed" # standalone, distributed, or replicated [cluster.nodes] # Initial cluster members for discovery seeds = [ "node1.geode.internal:7687", "node2.geode.internal:7687", "node3.geode.internal:7687" ] [cluster.node] id = "node1" role = "data" # leader, data, or coordinator data_dir = "/var/lib/geode/data" [cluster.consensus] protocol = "raft" election_timeout_ms = 1500 heartbeat_interval_ms = 150 </code></pre></div> <h4 id="cluster-formation" class="position-relative d-flex align-items-center group"> Cluster Formation <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="cluster-formation" aria-haspopup="dialog" aria-label="Share link: Cluster Formation"> Share link </button> </h4>When starting a Geode cluster, nodes discover each other and elect a leader: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Start first node (will become leader if cluster is new) ./geode serve --cluster-mode distributed \ --node-id node1 \ --cluster-seeds node1:7687,node2:7687,node3:7687 \ --listen 0.0.0.0:3141 \ --cluster-listen 0.0.0.0:7687 # Start additional nodes ./geode serve --cluster-mode distributed \ --node-id node2 \ --cluster-seeds node1:7687,node2:7687,node3:7687 \ --listen 0.0.0.0:3141 \ --cluster-listen 0.0.0.0:7687 </code></pre></div>Formation Process: <ol> <li>Nodes connect to seed addresses</li> <li>Cluster state is synchronized</li> <li>Leader election occurs via Raft</li> <li>Data partitions are assigned</li> <li>Cluster becomes operational</li> </ol> <h3 id="consensus-and-coordination" class="position-relative d-flex align-items-center group"> Consensus and Coordination <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="consensus-and-coordination" aria-haspopup="dialog" aria-label="Share link: Consensus and Coordination"> Share link </button> </h3> <h4 id="raft-consensus-protocol" class="position-relative d-flex align-items-center group"> Raft Consensus Protocol <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="raft-consensus-protocol" aria-haspopup="dialog" aria-label="Share link: Raft Consensus Protocol"> Share link </button> </h4>Geode uses the Raft consensus protocol for leader election and log replication: Leader Election: When the current leader fails or is unreachable, remaining nodes elect a new leader through voting. A node needs majority votes to become leader. Log Replication: All writes are first logged by the leader, then replicated to followers. A write is committed only after a majority of nodes have persisted it. Membership Changes: Adding or removing nodes is handled through joint consensus to ensure safety during transitions. <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- View cluster consensus status SELECT node_id, role, term, commit_index, last_applied FROM system.consensus_status; -- Check replication lag SELECT follower_id, leader_commit_index - follower_commit_index AS lag, last_heartbeat FROM system.replication_status; </code></pre></div>Consensus Configuration: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[cluster.consensus] protocol = "raft" # Election timeout (follower becomes candidate) election_timeout_ms = 1500 # Heartbeat interval (leader to followers) heartbeat_interval_ms = 150 # Maximum entries per append max_append_entries = 1000 # Snapshot threshold (log entries before snapshot) snapshot_threshold = 10000 </code></pre></div> <h4 id="distributed-transactions" class="position-relative d-flex align-items-center group"> Distributed Transactions <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="distributed-transactions" aria-haspopup="dialog" aria-label="Share link: Distributed Transactions"> Share link </button> </h4>Geode supports distributed ACID transactions using two-phase commit: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Distributed transaction spanning multiple shards BEGIN; -- Creates on shard-1 (user nodes) CREATE (u:User {id: 'user-123', name: 'Alice'}); -- Creates on shard-2 (product nodes) CREATE (p:Product {id: 'prod-456', name: 'Widget'}); -- Relationship spans both shards MATCH (u:User {id: 'user-123'}), (p:Product {id: 'prod-456'}) CREATE (u)-[:PURCHASED {date: datetime()}]->(p); COMMIT; </code></pre></div>Transaction Flow: <ol> <li>Prepare Phase: Coordinator asks all involved shards to prepare</li> <li>Vote: Each shard votes commit or abort</li> <li>Commit Phase: If all vote commit, coordinator broadcasts commit</li> <li>Completion: All shards apply changes and release locks</li> </ol> Transaction Configuration: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[transactions.distributed] enabled = true timeout_ms = 30000 max_retries = 3 retry_delay_ms = 100 # Two-phase commit settings prepare_timeout_ms = 10000 commit_timeout_ms = 10000 </code></pre></div> <h3 id="data-sharding-strategies" class="position-relative d-flex align-items-center group"> Data Sharding Strategies <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-sharding-strategies" aria-haspopup="dialog" aria-label="Share link: Data Sharding Strategies"> Share link </button> </h3> <h4 id="hash-based-sharding" class="position-relative d-flex align-items-center group"> Hash-Based Sharding <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="hash-based-sharding" aria-haspopup="dialog" aria-label="Share link: Hash-Based Sharding"> Share link </button> </h4>Distribute data based on property hash values: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Configure hash-based sharding CREATE GRAPH TYPE social_network ( User LABEL ( id STRING PRIMARY KEY, name STRING, email STRING ), Post LABEL ( id STRING PRIMARY KEY, content STRING, created_at DATETIME ) ) SHARD BY HASH(User.id, Post.id) SHARDS 16; </code></pre></div>Advantages: <ul> <li>Even data distribution</li> <li>Predictable shard location</li> <li>Simple implementation</li> </ul> Disadvantages: <ul> <li>Range queries span all shards</li> <li>No locality for related data</li> </ul> <h4 id="range-based-sharding" class="position-relative d-flex align-items-center group"> Range-Based Sharding <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="range-based-sharding" aria-haspopup="dialog" aria-label="Share link: Range-Based Sharding"> Share link </button> </h4>Partition data by property ranges: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Range-based sharding for time-series data CREATE GRAPH TYPE event_log ( Event LABEL ( id STRING PRIMARY KEY, timestamp DATETIME, type STRING, data JSON ) ) SHARD BY RANGE(Event.timestamp) ( PARTITION p2024q1 VALUES LESS THAN ('2024-04-01'), PARTITION p2024q2 VALUES LESS THAN ('2024-07-01'), PARTITION p2024q3 VALUES LESS THAN ('2024-10-01'), PARTITION p2024q4 VALUES LESS THAN ('2025-01-01'), PARTITION pmax VALUES LESS THAN MAXVALUE ); </code></pre></div>Advantages: <ul> <li>Efficient range queries</li> <li>Time-based data lifecycle management</li> <li>Partition pruning for queries</li> </ul> Disadvantages: <ul> <li>Potential hotspots on recent partitions</li> <li>Requires partition management</li> </ul> <h4 id="graph-aware-sharding" class="position-relative d-flex align-items-center group"> Graph-Aware Sharding <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="graph-aware-sharding" aria-haspopup="dialog" aria-label="Share link: Graph-Aware Sharding"> Share link </button> </h4>Colocate connected nodes to minimize cross-shard traversals: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Colocate users with their content CREATE GRAPH TYPE content_graph ( User LABEL ( id STRING PRIMARY KEY, tenant_id STRING ), Post LABEL ( id STRING PRIMARY KEY, author_id STRING ) ) SHARD BY COLOCATE(User.tenant_id, Post.author_id) SHARDS 8; </code></pre></div>Advantages: <ul> <li>Minimal cross-shard traversals</li> <li>Better query performance</li> <li>Reduced network overhead</li> </ul> Disadvantages: <ul> <li>Requires careful data modeling</li> <li>May cause uneven distribution</li> </ul> <h4 id="monitoring-shard-distribution" class="position-relative d-flex align-items-center group"> Monitoring Shard Distribution <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="monitoring-shard-distribution" aria-haspopup="dialog" aria-label="Share link: Monitoring Shard Distribution"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Check shard balance SELECT shard_id, node_count, edge_count, data_size_mb, primary_node FROM system.shard_statistics ORDER BY data_size_mb DESC; -- Identify cross-shard relationships SELECT relationship_type, COUNT(*) as total, SUM(CASE WHEN source_shard != target_shard THEN 1 ELSE 0 END) as cross_shard, ROUND(100.0 * SUM(CASE WHEN source_shard != target_shard THEN 1 ELSE 0 END) / COUNT(*), 2) as cross_shard_pct FROM system.relationship_distribution GROUP BY relationship_type; </code></pre></div> <h3 id="distributed-query-execution" class="position-relative d-flex align-items-center group"> Distributed Query Execution <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="distributed-query-execution" aria-haspopup="dialog" aria-label="Share link: Distributed Query Execution"> Share link </button> </h3> <h4 id="query-planning" class="position-relative d-flex align-items-center group"> Query Planning <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="query-planning" aria-haspopup="dialog" aria-label="Share link: Query Planning"> Share link </button> </h4>The query coordinator analyzes queries and generates distributed execution plans: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Distributed query EXPLAIN MATCH (u:User {country: 'USA'})-[:FOLLOWS]->(friend:User)-[:POSTED]->(p:Post) WHERE p.created_at > datetime() - duration('P7D') RETURN u.name, friend.name, p.title LIMIT 100; </code></pre></div>Execution Plan: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">DistributedQueryPlan: 1. ScatterGather: Filter User nodes (country = 'USA') - Shards: [1, 3, 5, 7, 9, 11, 13, 15] (based on hash distribution) 2. LocalExpand: (User)-[:FOLLOWS]->(friend) - Colocated: 78%, Cross-shard: 22% 3. RemoteFetch: Fetch cross-shard friends - Estimated remote calls: 2,340 4. LocalExpand: (friend)-[:POSTED]->(Post) 5. Filter: p.created_at > threshold 6. Gather: Aggregate results at coordinator 7. Limit: 100 Estimated Cost: 12,450 </code></pre></div> <h4 id="query-routing-strategies" class="position-relative d-flex align-items-center group"> Query Routing Strategies <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="query-routing-strategies" aria-haspopup="dialog" aria-label="Share link: Query Routing Strategies"> Share link </button> </h4>Scatter-Gather: Send query to all relevant shards, aggregate results <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Scatter-gather for aggregation MATCH (p:Product) RETURN p.category, COUNT(*) as count, AVG(p.price) as avg_price GROUP BY p.category; -- Runs on all shards, coordinator aggregates </code></pre></div>Directed Query: Route to specific shard based on filter <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Directed query with shard key MATCH (u:User {id: 'user-123'}) RETURN u; -- Routes directly to shard containing user-123 </code></pre></div>Broadcast Query: Send to all shards for global operations <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Broadcast for schema operations CREATE INDEX user_email ON User(email); -- Applied on all shards </code></pre></div> <h4 id="cross-shard-traversals" class="position-relative d-flex align-items-center group"> Cross-Shard Traversals <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="cross-shard-traversals" aria-haspopup="dialog" aria-label="Share link: Cross-Shard Traversals"> Share link </button> </h4>When traversals cross shard boundaries, Geode optimizes communication: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">from geode_client import Client async def efficient_traversal(): """Geode automatically optimizes cross-shard traversals""" client = Client(host="coordinator.geode.internal", port=3141) async with client.connection() as conn: # Geode batches remote fetches for efficiency result, _ = await conn.query(""" MATCH (u:User {id: $id})-[:FOLLOWS*1..3]->(friend) RETURN DISTINCT friend.id, friend.name """, {"id": "user-123"}) return result.rows </code></pre></div>Optimization Techniques: <ul> <li>Batch Remote Fetches: Collect all needed remote node IDs, fetch in batches</li> <li>Prefetch Hints: Predict likely traversal paths, prefetch data</li> <li>Query Caching: Cache frequently traversed paths</li> <li>Bloom Filters: Quickly determine if node exists on shard</li> </ul> <h3 id="fault-tolerance-and-recovery" class="position-relative d-flex align-items-center group"> Fault Tolerance and Recovery <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="fault-tolerance-and-recovery" aria-haspopup="dialog" aria-label="Share link: Fault Tolerance and Recovery"> Share link </button> </h3> <h4 id="handling-node-failures" class="position-relative d-flex align-items-center group"> Handling Node Failures <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="handling-node-failures" aria-haspopup="dialog" aria-label="Share link: Handling Node Failures"> Share link </button> </h4>Geode automatically detects and recovers from node failures: Failure Detection: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[cluster.health] # Failure detection settings heartbeat_interval_ms = 100 heartbeat_timeout_ms = 1000 failure_detection_threshold = 3 # Missed heartbeats before suspected </code></pre></div>Automatic Failover: <ol> <li>Failure detected via missed heartbeats</li> <li>Leader marks node as unavailable</li> <li>Replicas promoted to primary for affected shards</li> <li>Clients automatically redirect to new primaries</li> <li>Rebalancing triggered when replacement node joins</li> </ol> <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Monitor node health SELECT node_id, status, last_heartbeat, shard_count, is_leader FROM system.cluster_nodes; -- View failover history SELECT timestamp, event_type, affected_node, affected_shards, recovery_time_ms FROM system.failover_events ORDER BY timestamp DESC LIMIT 20; </code></pre></div> <h4 id="network-partition-handling" class="position-relative d-flex align-items-center group"> Network Partition Handling <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="network-partition-handling" aria-haspopup="dialog" aria-label="Share link: Network Partition Handling"> Share link </button> </h4>Geode handles network partitions following the Raft protocol: Majority Partition: Continues operating, elects new leader if needed Minority Partition: Becomes read-only, rejects writes <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[cluster.partition] # Partition handling strategy strategy = "majority" # majority, all, or custom # Read behavior during partition allow_stale_reads = false stale_read_timeout_ms = 5000 # Write behavior require_majority_ack = true </code></pre></div> <h4 id="data-replication" class="position-relative d-flex align-items-center group"> Data Replication <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-replication" aria-haspopup="dialog" aria-label="Share link: Data Replication"> Share link </button> </h4>Configure replication for durability and availability: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[replication] # Number of replicas per shard factor = 3 # Synchronous vs asynchronous mode = "sync" # sync or async # Replica placement placement = "rack-aware" # spread replicas across racks # Read preference read_preference = "primary" # primary, secondary, or nearest </code></pre></div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Check replication status SELECT shard_id, primary_node, replica_nodes, replication_lag_ms, last_sync FROM system.shard_replication; </code></pre></div> <h3 id="monitoring-distributed-operations" class="position-relative d-flex align-items-center group"> Monitoring Distributed Operations <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="monitoring-distributed-operations" aria-haspopup="dialog" aria-label="Share link: Monitoring Distributed Operations"> Share link </button> </h3> <h4 id="cluster-metrics" class="position-relative d-flex align-items-center group"> Cluster Metrics <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="cluster-metrics" aria-haspopup="dialog" aria-label="Share link: Cluster Metrics"> Share link </button> </h4>Key metrics for monitoring distributed Geode deployments: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Prometheus metrics endpoint curl http://coordinator:3141/metrics # Key distributed metrics geode_cluster_nodes_total{status="healthy"} 5 geode_cluster_nodes_total{status="unhealthy"} 0 geode_cluster_leader_elections_total 3 geode_cluster_replication_lag_seconds{shard="1"} 0.012 geode_distributed_queries_total{type="scatter_gather"} 45823 geode_cross_shard_bytes_total 1847293847 </code></pre></div>Grafana Dashboard Queries: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-yaml" data-lang="yaml"># Cluster health panel - title: "Cluster Health" targets: - expr: 'geode_cluster_nodes_total{status="healthy"}' legendFormat: "Healthy Nodes" # Replication lag panel - title: "Replication Lag" targets: - expr: 'histogram_quantile(0.99, rate(geode_cluster_replication_lag_seconds_bucket[5m]))' legendFormat: "p99 Lag" # Cross-shard traffic panel - title: "Cross-Shard Traffic" targets: - expr: 'rate(geode_cross_shard_bytes_total[5m])' legendFormat: "Bytes/sec" </code></pre></div> <h4 id="alerting-rules" class="position-relative d-flex align-items-center group"> Alerting Rules <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="alerting-rules" aria-haspopup="dialog" aria-label="Share link: Alerting Rules"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-yaml" data-lang="yaml"># Prometheus alerting rules groups: - name: geode_cluster rules: - alert: ClusterNodeDown expr: geode_cluster_nodes_total{status="unhealthy"} > 0 for: 1m labels: severity: critical annotations: summary: "Geode cluster node is unhealthy" - alert: HighReplicationLag expr: geode_cluster_replication_lag_seconds > 5 for: 5m labels: severity: warning annotations: summary: "Replication lag exceeds 5 seconds" - alert: FrequentLeaderElections expr: rate(geode_cluster_leader_elections_total[1h]) > 5 for: 10m labels: severity: warning annotations: summary: "Frequent leader elections detected" </code></pre></div> <h3 id="configuration-best-practices" class="position-relative d-flex align-items-center group"> Configuration Best Practices <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="configuration-best-practices" aria-haspopup="dialog" aria-label="Share link: Configuration Best Practices"> Share link </button> </h3> <h4 id="network-configuration" class="position-relative d-flex align-items-center group"> Network Configuration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="network-configuration" aria-haspopup="dialog" aria-label="Share link: Network Configuration"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[cluster.network] # Separate client and cluster traffic client_port = 3141 cluster_port = 7687 # Timeouts connect_timeout_ms = 5000 request_timeout_ms = 30000 # Keep-alive keepalive_interval_ms = 10000 keepalive_timeout_ms = 30000 # TLS for inter-node communication tls_enabled = true tls_cert_file = "/etc/geode/cluster.crt" tls_key_file = "/etc/geode/cluster.key" tls_ca_file = "/etc/geode/ca.crt" </code></pre></div> <h4 id="resource-limits" class="position-relative d-flex align-items-center group"> Resource Limits <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="resource-limits" aria-haspopup="dialog" aria-label="Share link: Resource Limits"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[cluster.resources] # Per-node limits max_concurrent_queries = 1000 max_transaction_size_mb = 100 max_cross_shard_batch_size = 10000 # Memory allocation node_memory_limit_gb = 32 query_memory_limit_gb = 8 replication_buffer_mb = 512 </code></pre></div> <h4 id="deployment-topology" class="position-relative d-flex align-items-center group"> Deployment Topology <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="deployment-topology" aria-haspopup="dialog" aria-label="Share link: Deployment Topology"> Share link </button> </h4>Three-Node Cluster (Minimum for HA): <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">Node 1: Leader + Data (Shard 1-5 primary, 11-16 replica) Node 2: Data (Shard 6-10 primary, 1-5 replica) Node 3: Data (Shard 11-16 primary, 6-10 replica) </code></pre></div>Five-Node Cluster (Better fault tolerance): <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">Node 1: Leader + Coordinator Node 2-5: Data nodes (4 shards primary each, 2 replicas) </code></pre></div>Production Cluster (Dedicated roles): <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">Coordinators: 2 nodes (active-passive) Data Nodes: 6+ nodes (3 replicas per shard) Load Balancer: External LB for client traffic </code></pre></div> <h3 id="best-practices" class="position-relative d-flex align-items-center group"> Best Practices <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="best-practices" aria-haspopup="dialog" aria-label="Share link: Best Practices"> Share link </button> </h3> <h4 id="data-modeling-for-distribution" class="position-relative d-flex align-items-center group"> Data Modeling for Distribution <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-modeling-for-distribution" aria-haspopup="dialog" aria-label="Share link: Data Modeling for Distribution"> Share link </button> </h4><ol> <li>Choose shard keys carefully: Select properties with even distribution</li> <li>Colocate related data: Keep frequently traversed relationships local</li> <li>Avoid hotspots: Distribute high-traffic nodes across shards</li> <li>Plan for growth: Choose sharding strategy that scales</li> </ol> <h4 id="query-optimization" class="position-relative d-flex align-items-center group"> Query Optimization <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="query-optimization" aria-haspopup="dialog" aria-label="Share link: Query Optimization"> Share link </button> </h4><ol> <li>Filter on shard key first: Enables directed queries</li> <li>Limit traversal depth: Deep traversals amplify cross-shard calls</li> <li>Use aggregation pushdown: Aggregate locally before gathering</li> <li>Cache hot traversals: Use application-level caching for popular paths</li> </ol> <h4 id="operations" class="position-relative d-flex align-items-center group"> Operations <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="operations" aria-haspopup="dialog" aria-label="Share link: Operations"> Share link </button> </h4><ol> <li>Monitor replication lag: Keep lag under SLA thresholds</li> <li>Test failover regularly: Verify automatic recovery works</li> <li>Plan capacity: Add nodes before reaching limits</li> <li>Rolling upgrades: Upgrade one node at a time</li> </ol> <h3 id="related-topics" class="position-relative d-flex align-items-center group"> Related Topics <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="related-topics" aria-haspopup="dialog" aria-label="Share link: Related Topics"> Share link </button> </h3><ul> <li><a href="/tags/high-availability/" >High Availability</a> - Configuring HA deployments</li> <li><a href="/tags/clustering/" >Clustering</a> - Cluster setup and management</li> <li><a href="/tags/recovery/" >Recovery</a> - Disaster recovery procedures</li> <li><a href="/tags/scaling/" >Scaling</a> - Horizontal and vertical scaling</li> <li><a href="/tags/performance/" >Performance</a> - Distributed query optimization</li> <li><a href="/tags/deployment/" >Deployment</a> - Production deployment patterns</li> </ul> <h3 id="further-reading" class="position-relative d-flex align-items-center group"> Further Reading <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="further-reading" aria-haspopup="dialog" aria-label="Share link: Further Reading"> Share link </button> </h3><ul> <li>Distributed Systems Architecture Guide</li> <li>Raft Consensus Protocol Overview</li> <li>Sharding Strategy Selection Guide</li> <li>Cross-Shard Query Optimization</li> <li>Network Partition Handling</li> <li>Cluster Operations Runbook</li> </ul>

Related Articles