Storage Engine

<h2 id="storage-engine-in-geode" class="position-relative d-flex align-items-center group"> Storage Engine in Geode <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="storage-engine-in-geode" aria-haspopup="dialog" aria-label="Share link: Storage Engine in Geode"> Share link </button> </h2><div id="headingShareModal" class="heading-share-modal" role="dialog" aria-modal="true" aria-labelledby="headingShareTitle" hidden> <div class="hsm-dialog" role="document"> <div class="hsm-header"> <h2 id="headingShareTitle" class="h6 mb-0 fw-bold">Share this section</h2> <button type="button" class="hsm-close" aria-label="Close"> </button> </div> <div class="hsm-body"> <label for="headingShareInput" class="form-label small text-muted mb-1 text-uppercase fw-bold" style="font-size: 0.7rem; letter-spacing: 0.5px;">Permalink</label> <div class="input-group mb-4 hsm-url-group"> <input id="headingShareInput" type="text" class="form-control font-monospace" readonly aria-readonly="true" style="font-size: 0.85rem;" /> <button class="btn btn-primary hsm-copy" type="button" aria-label="Copy" title="Copy"> </button> </div> <div class="small fw-bold mb-2 text-muted text-uppercase" style="font-size: 0.7rem; letter-spacing: 0.5px;">Share via</div> <div class="hsm-share-grid"> <a id="share-twitter" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Twitter </a> <a id="share-linkedin" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> LinkedIn </a> <a id="share-facebook" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Facebook </a> </div> </div> </div> </div> <style> .heading-share-modal { position: fixed; inset: 0; display: flex; justify-content: center; align-items: center; background: rgba(0, 0, 0, 0.6); z-index: 1050; padding: 1rem; backdrop-filter: blur(4px); -webkit-backdrop-filter: blur(4px); } .heading-share-modal[hidden] { display: none !important; } .hsm-dialog { max-width: 420px; width: 100%; background: var(--bs-body-bg, #fff); color: var(--bs-body-color, #212529); border: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); border-radius: 1rem; box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.25); overflow: hidden; animation: hsm-fade-in 0.2s ease-out; } @keyframes hsm-fade-in { from { opacity: 0; transform: scale(0.95); } to { opacity: 1; transform: scale(1); } } [data-bs-theme="dark"] .hsm-dialog { background: #1e293b; border-color: rgba(255,255,255,0.1); color: #f8f9fa; } .hsm-header { display: flex; justify-content: space-between; align-items: center; padding: 1rem 1.5rem; border-bottom: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); background: rgba(0,0,0,0.02); } [data-bs-theme="dark"] .hsm-header { background: rgba(255,255,255,0.02); border-color: rgba(255,255,255,0.1); } .hsm-close { background: transparent; border: none; color: inherit; opacity: 0.5; padding: 0.25rem 0.5rem; border-radius: 0.25rem; font-size: 1.2rem; line-height: 1; transition: opacity 0.2s; } .hsm-close:hover { opacity: 1; } .hsm-body { padding: 1.5rem; } .hsm-url-group { display: flex !important; align-items: stretch; } .hsm-url-group .form-control { flex: 1; min-width: 0; margin: 0; background: var(--bs-secondary-bg, #f8f9fa); border-color: var(--bs-border-color, #dee2e6); border-top-right-radius: 0; border-bottom-right-radius: 0; height: 42px; } .hsm-url-group .btn { flex: 0 0 auto; margin: 0; margin-left: -1px; border-top-left-radius: 0; border-bottom-left-radius: 0; height: 42px; display: flex; align-items: center; justify-content: center; padding: 0 1.25rem; z-index: 2; } [data-bs-theme="dark"] .hsm-url-group .form-control { background: #0f172a; border-color: #334155; color: #e2e8f0; } .hsm-share-grid { display: flex; flex-direction: column; gap: 0.5rem; } .hsm-share-grid .btn { display: flex; align-items: center; justify-content: center; font-size: 0.9rem; padding: 0.6rem; border-color: var(--bs-border-color); width: 100%; } [data-bs-theme="dark"] .hsm-share-grid .btn { color: #e2e8f0; border-color: #475569; } [data-bs-theme="dark"] .hsm-share-grid .btn:hover { background: #334155; border-color: #cbd5e1; } </style> <script> (function(){ const modal = document.getElementById('headingShareModal'); if(!modal) return; const input = modal.querySelector('#headingShareInput'); const copyBtn = modal.querySelector('.hsm-copy'); const twitter = modal.querySelector('#share-twitter'); const linkedin = modal.querySelector('#share-linkedin'); const facebook = modal.querySelector('#share-facebook'); const closeBtn = modal.querySelector('.hsm-close'); let lastFocus=null; let trapBound=false; function buildUrl(id){ return window.location.origin + window.location.pathname + '#' + id; } function isOpen(){ return !modal.hasAttribute('hidden'); } function hydrate(id){ const url=buildUrl(id); input.value=url; const enc=encodeURIComponent(url); const text=encodeURIComponent(document.title); if(twitter) twitter.href=`https://twitter.com/intent/tweet?url=${enc}&text=${text}`; if(linkedin) linkedin.href=`https://www.linkedin.com/sharing/share-offsite/?url=${enc}`; if(facebook) facebook.href=`https://www.facebook.com/sharer/sharer.php?u=${enc}`; } function openModal(id){ lastFocus=document.activeElement; hydrate(id); if(!isOpen()){ modal.removeAttribute('hidden'); } requestAnimationFrame(()=>{ input.focus(); }); trapFocus(); } function closeModal(){ if(!isOpen()) return; modal.setAttribute('hidden',''); if(lastFocus && typeof lastFocus.focus==='function') lastFocus.focus(); } function copyCurrent(){ try{ navigator.clipboard.writeText(input.value).then(()=>feedback(true),()=>fallback()); } catch(e){ fallback(); } } function fallback(){ input.select(); try{ document.execCommand('copy'); feedback(true);}catch(e){ feedback(false);} } function feedback(ok){ if(!copyBtn) return; const icon=copyBtn.querySelector('i'); if(!icon) return; const prev=copyBtn.getAttribute('data-prev')||icon.className; if(!copyBtn.getAttribute('data-prev')) copyBtn.setAttribute('data-prev',prev); icon.className= ok ? 'fa-duotone fa-clipboard-check':'fa-duotone fa-circle-exclamation'; setTimeout(()=>{ icon.className=prev; },1800); } function handleShareClick(e){ e.preventDefault(); const btn=e.currentTarget; const id=btn.getAttribute('data-share-target'); if(id) openModal(id); } function bindShareButtons(){ document.querySelectorAll('.h-share').forEach(btn=>{ if(!btn.dataset.hShareBound){ btn.addEventListener('click', handleShareClick); btn.dataset.hShareBound='1'; } }); } bindShareButtons(); if(document.readyState==='loading'){ document.addEventListener('DOMContentLoaded', bindShareButtons); } else { requestAnimationFrame(bindShareButtons); } document.addEventListener('click', function(e){ const shareBtn=e.target.closest && e.target.closest('.h-share'); if(shareBtn && !shareBtn.dataset.hShareBound){ handleShareClick.call(shareBtn, e); } }, true); document.addEventListener('click', e=>{ if(e.target===modal) closeModal(); if(e.target.closest && e.target.closest('.hsm-close')){ e.preventDefault(); closeModal(); } if(copyBtn && (e.target===copyBtn || (e.target.closest && e.target.closest('.hsm-copy')))) { e.preventDefault(); copyCurrent(); } }); document.addEventListener('keydown', e=>{ if(e.key==='Escape' && isOpen()) closeModal(); }); function trapFocus(){ if(trapBound) return; trapBound=true; modal.addEventListener('keydown', f=>{ if(f.key==='Tab' && isOpen()){ const focusable=[...modal.querySelectorAll('a[href],button,input,textarea,select,[tabindex]:not([tabindex="-1"])')].filter(el=>!el.hasAttribute('disabled')); if(!focusable.length) return; const first=focusable[0]; const last=focusable[focusable.length-1]; if(f.shiftKey && document.activeElement===first){ f.preventDefault(); last.focus(); } else if(!f.shiftKey && document.activeElement===last){ f.preventDefault(); first.focus(); } } }); } if(closeBtn) closeBtn.addEventListener('click', e=>{ e.preventDefault(); closeModal(); }); })(); </script>The storage engine is the foundation of any database system, responsible for durably persisting data, managing memory, and providing efficient access patterns. Geode’s storage engine is purpose-built for graph workloads, optimizing for relationship traversals, flexible schemas, and ACID transactions. This guide explores Geode’s storage architecture, configuration options, and best practices for optimal storage performance. <h3 id="storage-architecture-overview" class="position-relative d-flex align-items-center group"> Storage Architecture Overview <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="storage-architecture-overview" aria-haspopup="dialog" aria-label="Share link: Storage Architecture Overview"> Share link </button> </h3> <h4 id="design-principles" class="position-relative d-flex align-items-center group"> Design Principles <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="design-principles" aria-haspopup="dialog" aria-label="Share link: Design Principles"> Share link </button> </h4>Geode’s storage engine follows key principles: Graph-Native Design: Optimized for adjacency lookups and traversals, not just point queries Memory-Mapped I/O: Efficient use of OS page cache for frequently accessed data Write-Ahead Logging: Durability without sacrificing write performance Copy-on-Write MVCC: Non-blocking reads during concurrent modifications Tiered Storage: Hot/warm/cold data management for cost efficiency <h4 id="high-level-architecture" class="position-relative d-flex align-items-center group"> High-Level Architecture <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="high-level-architecture" aria-haspopup="dialog" aria-label="Share link: High-Level Architecture"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">┌─────────────────────────────────────────────────────────────┐ │ Query Engine │ └──────────────────────────┬──────────────────────────────────┘ │ ┌──────────────────────────▼──────────────────────────────────┐ │ Transaction Manager │ │ (MVCC, Locking, Isolation) │ └──────────────────────────┬──────────────────────────────────┘ │ ┌──────────────────────────▼──────────────────────────────────┐ │ Buffer Pool │ │ (Page Cache, Dirty Page Management) │ └──────────────────────────┬──────────────────────────────────┘ │ ┌────────────────┼────────────────┐ ▼ ▼ ▼ ┌─────────────────┐ ┌─────────────┐ ┌─────────────────┐ │ Index Files │ │ Data Files │ │ WAL Files │ │ (B-tree) │ │ (Pages) │ │ (Sequential) │ └─────────────────┘ └─────────────┘ └─────────────────┘ </code></pre></div> <h3 id="data-file-organization" class="position-relative d-flex align-items-center group"> Data File Organization <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-file-organization" aria-haspopup="dialog" aria-label="Share link: Data File Organization"> Share link </button> </h3> <h4 id="page-structure" class="position-relative d-flex align-items-center group"> Page Structure <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="page-structure" aria-haspopup="dialog" aria-label="Share link: Page Structure"> Share link </button> </h4>Data is organized into fixed-size pages (default 16 KB): <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gdscript3" data-lang="gdscript3">Page Layout (16 KB): ┌──────────────────────────────────────────────────────────┐ │ Page Header (64 bytes) │ │ - Page ID (8 bytes) │ │ - Page Type (2 bytes): DATA, INDEX, OVERFLOW, FREE │ │ - LSN (8 bytes): Log Sequence Number │ │ - Checksum (4 bytes) │ │ - Free Space Offset (2 bytes) │ │ - Item Count (2 bytes) │ │ - Flags (2 bytes) │ │ - Reserved (36 bytes) │ ├──────────────────────────────────────────────────────────┤ │ Item Pointers (variable) │ │ - Offset (2 bytes) + Length (2 bytes) per item │ ├──────────────────────────────────────────────────────────┤ │ │ │ Free Space │ │ │ ├──────────────────────────────────────────────────────────┤ │ Items (variable) │ │ - Node records, edge records, or property data │ └──────────────────────────────────────────────────────────┘ </code></pre></div>Page Configuration: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.pages] size_kb = 16 # Page size (8, 16, or 32 KB) alignment = 4096 # Disk alignment (usually 4K) checksum = "crc32c" # Checksum algorithm </code></pre></div> <h4 id="node-storage" class="position-relative d-flex align-items-center group"> Node Storage <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="node-storage" aria-haspopup="dialog" aria-label="Share link: Node Storage"> Share link </button> </h4>Nodes are stored with their labels and properties: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gdscript3" data-lang="gdscript3">Node Record: ┌─────────────────────────────────────────────────────────┐ │ Node ID (8 bytes) │ │ Label Bitmap (8 bytes) - Up to 64 labels per node │ │ Property Count (2 bytes) │ │ First Edge Pointer (8 bytes) - Outgoing edges │ │ Properties (variable): │ │ - Key ID (4 bytes) │ │ - Type (1 byte) │ │ - Value (variable) │ │ - ... │ │ Overflow Pointer (8 bytes) - For large properties │ └─────────────────────────────────────────────────────────┘ </code></pre></div> <h4 id="edge-storage" class="position-relative d-flex align-items-center group"> Edge Storage <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="edge-storage" aria-haspopup="dialog" aria-label="Share link: Edge Storage"> Share link </button> </h4>Edges connect nodes with relationship data: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gdscript3" data-lang="gdscript3">Edge Record: ┌─────────────────────────────────────────────────────────┐ │ Edge ID (8 bytes) │ │ Source Node ID (8 bytes) │ │ Target Node ID (8 bytes) │ │ Relationship Type ID (4 bytes) │ │ Next Outgoing Edge (8 bytes) - Linked list │ │ Next Incoming Edge (8 bytes) - Linked list │ │ Property Count (2 bytes) │ │ Properties (variable) │ └─────────────────────────────────────────────────────────┘ </code></pre></div> <h4 id="file-layout" class="position-relative d-flex align-items-center group"> File Layout <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="file-layout" aria-haspopup="dialog" aria-label="Share link: File Layout"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gdscript3" data-lang="gdscript3">Data Directory Structure: /var/lib/geode/data/ ├── base/ # Main database files │ ├── nodes.dat # Node data pages │ ├── edges.dat # Edge data pages │ ├── properties.dat # Large property overflow │ └── free_space.map # Free space tracking ├── indexes/ # Index files │ ├── node_labels.idx # Label index │ ├── edge_types.idx # Relationship type index │ └── property_*.idx # Property indexes ├── wal/ # Write-ahead log │ ├── 000000010000000000000001 │ ├── 000000010000000000000002 │ └── ... ├── system/ # System catalog │ ├── schema.dat # Schema definitions │ ├── statistics.dat # Query statistics │ └── config.dat # Runtime configuration └── temp/ # Temporary files └── sort_*.tmp # Sort spill files </code></pre></div> <h3 id="write-ahead-logging-wal" class="position-relative d-flex align-items-center group"> Write-Ahead Logging (WAL) <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="write-ahead-logging-wal" aria-haspopup="dialog" aria-label="Share link: Write-Ahead Logging (WAL)"> Share link </button> </h3> <h4 id="wal-architecture" class="position-relative d-flex align-items-center group"> WAL Architecture <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="wal-architecture" aria-haspopup="dialog" aria-label="Share link: WAL Architecture"> Share link </button> </h4>All modifications are written to the WAL before applying to data files: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">Write Flow: 1. Transaction begins 2. Modifications logged to WAL buffer 3. WAL buffer flushed to disk (fsync) 4. Changes applied to buffer pool (memory) 5. Transaction commits 6. Background: dirty pages flushed to data files </code></pre></div>WAL Record Format: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gdscript3" data-lang="gdscript3">WAL Record: ┌─────────────────────────────────────────────────────────┐ │ LSN (8 bytes) - Log Sequence Number │ │ Transaction ID (8 bytes) │ │ Record Type (1 byte): INSERT, UPDATE, DELETE, COMMIT │ │ Table ID (4 bytes) │ │ Record Length (4 bytes) │ │ Before Image (variable) - For rollback │ │ After Image (variable) - For replay │ │ Checksum (4 bytes) │ └─────────────────────────────────────────────────────────┘ </code></pre></div> <h4 id="wal-configuration" class="position-relative d-flex align-items-center group"> WAL Configuration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="wal-configuration" aria-haspopup="dialog" aria-label="Share link: WAL Configuration"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.wal] enabled = true directory = "/var/lib/geode/wal" # Segment size (WAL files) segment_size_mb = 64 # Sync mode sync_mode = "fsync" # fsync, fdatasync, or async # Buffer size buffer_size_mb = 16 # Checkpoint settings checkpoint_interval_seconds = 300 checkpoint_threshold_mb = 1024 # Archiving archive_enabled = true archive_command = "/etc/geode/archive_wal.sh %f %p" </code></pre></div> <h4 id="wal-sync-modes" class="position-relative d-flex align-items-center group"> WAL Sync Modes <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="wal-sync-modes" aria-haspopup="dialog" aria-label="Share link: WAL Sync Modes"> Share link </button> </h4>fsync (default): Full durability, slightly slower <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.wal] sync_mode = "fsync" </code></pre></div>fdatasync: Faster, skips metadata sync <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.wal] sync_mode = "fdatasync" </code></pre></div>async: Best performance, risk of data loss on crash <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.wal] sync_mode = "async" sync_interval_ms = 100 # Periodic sync </code></pre></div> <h4 id="monitoring-wal" class="position-relative d-flex align-items-center group"> Monitoring WAL <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="monitoring-wal" aria-haspopup="dialog" aria-label="Share link: Monitoring WAL"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- WAL statistics SELECT current_wal_lsn, last_checkpoint_lsn, wal_size_mb, wal_write_rate_mb_per_sec, checkpoint_in_progress FROM system.wal_stats; -- WAL file status SELECT segment_name, size_mb, start_lsn, end_lsn, archived, archived_at FROM system.wal_segments ORDER BY start_lsn DESC LIMIT 10; </code></pre></div> <h3 id="buffer-pool-management" class="position-relative d-flex align-items-center group"> Buffer Pool Management <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="buffer-pool-management" aria-haspopup="dialog" aria-label="Share link: Buffer Pool Management"> Share link </button> </h3> <h4 id="buffer-pool-architecture" class="position-relative d-flex align-items-center group"> Buffer Pool Architecture <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="buffer-pool-architecture" aria-haspopup="dialog" aria-label="Share link: Buffer Pool Architecture"> Share link </button> </h4>The buffer pool caches data pages in memory: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">Buffer Pool: ┌─────────────────────────────────────────────────────────┐ │ Hash Table (Page ID -> Buffer Index) │ ├─────────────────────────────────────────────────────────┤ │ Buffer Frames: │ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ │ Frame 0 │ │ Frame 1 │ │ Frame 2 │ ... │ │ │ Page 42 │ │ Page 17 │ │ Page 891 │ │ │ │ Dirty │ │ Clean │ │ Dirty │ │ │ │ Pinned:2 │ │ Pinned:0 │ │ Pinned:1 │ │ │ └──────────┘ └──────────┘ └──────────┘ │ ├─────────────────────────────────────────────────────────┤ │ LRU List (eviction candidates) │ ├─────────────────────────────────────────────────────────┤ │ Dirty Page List (checkpoint candidates) │ └─────────────────────────────────────────────────────────┘ </code></pre></div> <h4 id="configuration" class="position-relative d-flex align-items-center group"> Configuration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="configuration" aria-haspopup="dialog" aria-label="Share link: Configuration"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.buffer_pool] # Total buffer pool size size_mb = 4096 # 4 GB # Percentage for different purposes data_cache_percent = 70 index_cache_percent = 25 temp_buffer_percent = 5 # Eviction policy eviction_policy = "lru" # lru, lru-k, or clock # Background writer background_writer_enabled = true background_writer_interval_ms = 100 background_writer_batch_size = 64 # Prefetching prefetch_enabled = true prefetch_distance = 32 # Pages to read ahead </code></pre></div> <h4 id="page-replacement" class="position-relative d-flex align-items-center group"> Page Replacement <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="page-replacement" aria-haspopup="dialog" aria-label="Share link: Page Replacement"> Share link </button> </h4>Geode uses LRU-K for intelligent page eviction: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Buffer pool statistics SELECT total_pages, used_pages, dirty_pages, hit_ratio, evictions_per_sec, reads_per_sec, writes_per_sec FROM system.buffer_pool_stats; -- Per-table buffer usage SELECT table_name, cached_pages, cached_mb, hit_ratio FROM system.buffer_pool_by_table ORDER BY cached_mb DESC LIMIT 10; </code></pre></div> <h3 id="index-storage" class="position-relative d-flex align-items-center group"> Index Storage <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="index-storage" aria-haspopup="dialog" aria-label="Share link: Index Storage"> Share link </button> </h3> <h4 id="b-tree-indexes" class="position-relative d-flex align-items-center group"> B-Tree Indexes <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="b-tree-indexes" aria-haspopup="dialog" aria-label="Share link: B-Tree Indexes"> Share link </button> </h4>Primary index structure for most lookups: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">B-Tree Structure: ┌─────────────┐ │ Root Node │ │ [50, 100] │ └──────┬──────┘ ┌────────────┼────────────┐ ▼ ▼ ▼ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ < 50 │ │ 50-100 │ │ > 100 │ │[10,25] │ │[75,90] │ │[125,150]│ └────┬────┘ └────┬────┘ └────┬────┘ ... ... ... ▼ ▼ ▼ ┌─────────┐ ┌─────────┐ ┌─────────┐ │Leaf Page│ │Leaf Page│ │Leaf Page│ │Key→Value│ │Key→Value│ │Key→Value│ └─────────┘ └─────────┘ └─────────┘ </code></pre></div>B-Tree Configuration: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.indexes.btree] # Fill factor for leaf pages fill_factor = 90 # Percent # Node split strategy split_strategy = "balanced" # balanced or right_biased # Bulk loading optimization bulk_load_factor = 95 </code></pre></div> <h4 id="index-file-format" class="position-relative d-flex align-items-center group"> Index File Format <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="index-file-format" aria-haspopup="dialog" aria-label="Share link: Index File Format"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">Index Page: ┌─────────────────────────────────────────────────────────┐ │ Page Header │ │ - Page Type: INDEX_INTERNAL or INDEX_LEAF │ │ - Level: 0 for leaf, 1+ for internal │ │ - Key Count │ │ - Right Sibling Pointer │ ├─────────────────────────────────────────────────────────┤ │ Keys and Pointers: │ │ Internal: [Key1][Ptr1][Key2][Ptr2]...[PtrN] │ │ Leaf: [Key1][Value1][Key2][Value2]... │ └─────────────────────────────────────────────────────────┘ </code></pre></div> <h4 id="vector-indexes-hnsw" class="position-relative d-flex align-items-center group"> Vector Indexes (HNSW) <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="vector-indexes-hnsw" aria-haspopup="dialog" aria-label="Share link: Vector Indexes (HNSW)"> Share link </button> </h4>For similarity search on embeddings: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.indexes.hnsw] # HNSW parameters m = 16 # Connections per layer ef_construction = 200 # Construction time quality ef_search = 50 # Search time quality # Memory mapping mmap_enabled = true preload = false # Load into memory on startup # Quantization for memory efficiency quantization = "none" # none, pq, or sq </code></pre></div> <h3 id="compaction-and-maintenance" class="position-relative d-flex align-items-center group"> Compaction and Maintenance <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="compaction-and-maintenance" aria-haspopup="dialog" aria-label="Share link: Compaction and Maintenance"> Share link </button> </h3> <h4 id="background-compaction" class="position-relative d-flex align-items-center group"> Background Compaction <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="background-compaction" aria-haspopup="dialog" aria-label="Share link: Background Compaction"> Share link </button> </h4>Geode performs continuous compaction to reclaim space: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.compaction] enabled = true # Compaction triggers dead_tuple_threshold = 20 # Percent dead tuples size_amplification_threshold = 1.5 # Compaction schedule schedule = "continuous" # continuous or scheduled scheduled_time = "03:00" # Resource limits max_concurrent_compactions = 2 throttle_mb_per_sec = 100 </code></pre></div> <h4 id="manual-maintenance" class="position-relative d-flex align-items-center group"> Manual Maintenance <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="manual-maintenance" aria-haspopup="dialog" aria-label="Share link: Manual Maintenance"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Trigger compaction ./geode admin compact --graph social_network # Analyze statistics ./geode admin analyze --graph social_network # Vacuum dead tuples ./geode admin vacuum --graph social_network # Rebuild indexes ./geode admin reindex --index user_email_idx </code></pre></div>Via GQL: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Compact a specific table CALL system.compact('User'); -- Update statistics ANALYZE User; -- Vacuum dead tuples VACUUM User; -- Rebuild index REINDEX INDEX user_email_idx; -- Check fragmentation SELECT table_name, live_tuples, dead_tuples, dead_tuple_ratio, last_vacuum, last_analyze FROM system.table_stats; </code></pre></div> <h4 id="checkpointing" class="position-relative d-flex align-items-center group"> Checkpointing <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="checkpointing" aria-haspopup="dialog" aria-label="Share link: Checkpointing"> Share link </button> </h4>Checkpoints write dirty pages to disk and advance the recovery point: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage.checkpoint] # Checkpoint triggers interval_seconds = 300 wal_size_mb = 1024 dirty_page_percent = 50 # Checkpoint behavior spread_writes = true # Spread I/O over time spread_duration_seconds = 60 # Monitoring log_checkpoints = true </code></pre></div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Checkpoint status SELECT checkpoint_start_time, checkpoint_end_time, duration_seconds, pages_written, wal_segments_removed FROM system.checkpoint_log ORDER BY checkpoint_start_time DESC LIMIT 5; -- Force checkpoint CHECKPOINT; </code></pre></div> <h3 id="storage-monitoring" class="position-relative d-flex align-items-center group"> Storage Monitoring <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="storage-monitoring" aria-haspopup="dialog" aria-label="Share link: Storage Monitoring"> Share link </button> </h3> <h4 id="key-metrics" class="position-relative d-flex align-items-center group"> Key Metrics <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="key-metrics" aria-haspopup="dialog" aria-label="Share link: Key Metrics"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Prometheus metrics curl http://localhost:3141/metrics | grep -E "geode_storage|geode_buffer|geode_wal" # Example metrics geode_storage_data_size_bytes 13421772800 geode_storage_index_size_bytes 2147483648 geode_storage_wal_size_bytes 536870912 geode_buffer_pool_hits_total 8472938 geode_buffer_pool_misses_total 234789 geode_buffer_pool_dirty_pages 1234 geode_wal_writes_total 847293 geode_wal_bytes_written_total 2147483648 geode_checkpoint_duration_seconds_sum 45.7 </code></pre></div> <h4 id="storage-health-queries" class="position-relative d-flex align-items-center group"> Storage Health Queries <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="storage-health-queries" aria-haspopup="dialog" aria-label="Share link: Storage Health Queries"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Overall storage statistics SELECT data_size_gb, index_size_gb, wal_size_gb, free_space_gb, total_size_gb, fragmentation_percent FROM system.storage_overview; -- Per-graph storage SELECT graph_name, node_count, edge_count, data_size_mb, index_size_mb FROM system.graph_storage ORDER BY data_size_mb DESC; -- Disk I/O statistics SELECT reads_per_sec, writes_per_sec, read_bytes_per_sec, write_bytes_per_sec, avg_read_latency_ms, avg_write_latency_ms FROM system.disk_io_stats; </code></pre></div> <h4 id="alerting-rules" class="position-relative d-flex align-items-center group"> Alerting Rules <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="alerting-rules" aria-haspopup="dialog" aria-label="Share link: Alerting Rules"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-yaml" data-lang="yaml"># Prometheus alerts for storage groups: - name: geode_storage_alerts rules: - alert: DiskSpaceLow expr: geode_storage_free_bytes / geode_storage_total_bytes < 0.1 for: 10m labels: severity: warning annotations: summary: "Disk space below 10%" - alert: DiskSpaceCritical expr: geode_storage_free_bytes / geode_storage_total_bytes < 0.05 for: 5m labels: severity: critical annotations: summary: "Disk space below 5%" - alert: BufferPoolHitRateLow expr: geode_buffer_pool_hit_ratio < 0.9 for: 15m labels: severity: warning annotations: summary: "Buffer pool hit rate below 90%" - alert: WALGrowthHigh expr: rate(geode_wal_bytes_written_total[5m]) > 100000000 for: 10m labels: severity: warning annotations: summary: "High WAL write rate" - alert: CheckpointTooLong expr: geode_checkpoint_duration_seconds > 300 for: 5m labels: severity: warning annotations: summary: "Checkpoint taking too long" </code></pre></div> <h3 id="storage-configuration-best-practices" class="position-relative d-flex align-items-center group"> Storage Configuration Best Practices <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="storage-configuration-best-practices" aria-haspopup="dialog" aria-label="Share link: Storage Configuration Best Practices"> Share link </button> </h3> <h4 id="hardware-recommendations" class="position-relative d-flex align-items-center group"> Hardware Recommendations <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="hardware-recommendations" aria-haspopup="dialog" aria-label="Share link: Hardware Recommendations"> Share link </button> </h4>SSDs: Strongly recommended for production <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage] # SSD-optimized settings disk_type = "ssd" page_size_kb = 16 read_ahead_kb = 256 </code></pre></div>NVMe: Best performance for write-heavy workloads <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage] disk_type = "nvme" page_size_kb = 16 io_depth = 32 </code></pre></div>HDDs: Only for archival/cold storage <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml">[storage] disk_type = "hdd" page_size_kb = 32 read_ahead_kb = 1024 sequential_read_threshold = 64 </code></pre></div> <h4 id="memory-sizing" class="position-relative d-flex align-items-center group"> Memory Sizing <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="memory-sizing" aria-haspopup="dialog" aria-label="Share link: Memory Sizing"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml"># Rule of thumb: buffer pool = 50-75% of available RAM [storage.buffer_pool] size_mb = 32768 # 32 GB for 48 GB system # Working set should fit in buffer pool # Monitor hit ratio and adjust </code></pre></div> <h4 id="file-system-settings" class="position-relative d-flex align-items-center group"> File System Settings <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="file-system-settings" aria-haspopup="dialog" aria-label="Share link: File System Settings"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Linux ext4 recommended settings mkfs.ext4 -O ^has_journal /dev/sdb1 # Disable journal (WAL handles durability) # Mount options mount -o noatime,nodiratime,data=writeback /dev/sdb1 /var/lib/geode # Disable transparent huge pages echo never > /sys/kernel/mm/transparent_hugepage/enabled # Increase file descriptors ulimit -n 65535 </code></pre></div> <h4 id="configuration-template" class="position-relative d-flex align-items-center group"> Configuration Template <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="configuration-template" aria-haspopup="dialog" aria-label="Share link: Configuration Template"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-toml" data-lang="toml"># Production storage configuration [storage] data_directory = "/var/lib/geode/data" temp_directory = "/var/lib/geode/temp" [storage.pages] size_kb = 16 checksum = "crc32c" [storage.buffer_pool] size_mb = 32768 eviction_policy = "lru" background_writer_enabled = true [storage.wal] directory = "/var/lib/geode/wal" segment_size_mb = 64 sync_mode = "fsync" checkpoint_interval_seconds = 300 [storage.compaction] enabled = true max_concurrent_compactions = 2 throttle_mb_per_sec = 100 [storage.indexes] fill_factor = 90 </code></pre></div> <h3 id="related-topics" class="position-relative d-flex align-items-center group"> Related Topics <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="related-topics" aria-haspopup="dialog" aria-label="Share link: Related Topics"> Share link </button> </h3><ul> <li><a href="/tags/performance/" >Performance</a> - Performance optimization</li> <li><a href="/tags/caching/" >Caching</a> - Caching strategies</li> <li><a href="/tags/backup/" >Backup</a> - Backup procedures</li> <li><a href="/tags/recovery/" >Recovery</a> - Recovery procedures</li> <li><a href="/tags/indexing/" >Indexing</a> - Index management</li> <li><a href="/tags/configuration/" >Configuration</a> - Server configuration</li> </ul> <h3 id="further-reading" class="position-relative d-flex align-items-center group"> Further Reading <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="further-reading" aria-haspopup="dialog" aria-label="Share link: Further Reading"> Share link </button> </h3><ul> <li>Storage Engine Architecture Deep Dive</li> <li>WAL Configuration Guide</li> <li>Buffer Pool Tuning</li> <li>Index Selection and Maintenance</li> <li>Storage Capacity Planning</li> <li>I/O Performance Optimization</li> </ul>

Popular

Related Articles