Data Migration

<h2 id="data-migration" class="position-relative d-flex align-items-center group"> Data Migration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-migration" aria-haspopup="dialog" aria-label="Share link: Data Migration"> Share link </button> </h2><div id="headingShareModal" class="heading-share-modal" role="dialog" aria-modal="true" aria-labelledby="headingShareTitle" hidden> <div class="hsm-dialog" role="document"> <div class="hsm-header"> <h2 id="headingShareTitle" class="h6 mb-0 fw-bold">Share this section</h2> <button type="button" class="hsm-close" aria-label="Close"> </button> </div> <div class="hsm-body"> <label for="headingShareInput" class="form-label small text-muted mb-1 text-uppercase fw-bold" style="font-size: 0.7rem; letter-spacing: 0.5px;">Permalink</label> <div class="input-group mb-4 hsm-url-group"> <input id="headingShareInput" type="text" class="form-control font-monospace" readonly aria-readonly="true" style="font-size: 0.85rem;" /> <button class="btn btn-primary hsm-copy" type="button" aria-label="Copy" title="Copy"> </button> </div> <div class="small fw-bold mb-2 text-muted text-uppercase" style="font-size: 0.7rem; letter-spacing: 0.5px;">Share via</div> <div class="hsm-share-grid"> <a id="share-twitter" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Twitter </a> <a id="share-linkedin" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> LinkedIn </a> <a id="share-facebook" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Facebook </a> </div> </div> </div> </div> <style> .heading-share-modal { position: fixed; inset: 0; display: flex; justify-content: center; align-items: center; background: rgba(0, 0, 0, 0.6); z-index: 1050; padding: 1rem; backdrop-filter: blur(4px); -webkit-backdrop-filter: blur(4px); } .heading-share-modal[hidden] { display: none !important; } .hsm-dialog { max-width: 420px; width: 100%; background: var(--bs-body-bg, #fff); color: var(--bs-body-color, #212529); border: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); border-radius: 1rem; box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.25); overflow: hidden; animation: hsm-fade-in 0.2s ease-out; } @keyframes hsm-fade-in { from { opacity: 0; transform: scale(0.95); } to { opacity: 1; transform: scale(1); } } [data-bs-theme="dark"] .hsm-dialog { background: #1e293b; border-color: rgba(255,255,255,0.1); color: #f8f9fa; } .hsm-header { display: flex; justify-content: space-between; align-items: center; padding: 1rem 1.5rem; border-bottom: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); background: rgba(0,0,0,0.02); } [data-bs-theme="dark"] .hsm-header { background: rgba(255,255,255,0.02); border-color: rgba(255,255,255,0.1); } .hsm-close { background: transparent; border: none; color: inherit; opacity: 0.5; padding: 0.25rem 0.5rem; border-radius: 0.25rem; font-size: 1.2rem; line-height: 1; transition: opacity 0.2s; } .hsm-close:hover { opacity: 1; } .hsm-body { padding: 1.5rem; } .hsm-url-group { display: flex !important; align-items: stretch; } .hsm-url-group .form-control { flex: 1; min-width: 0; margin: 0; background: var(--bs-secondary-bg, #f8f9fa); border-color: var(--bs-border-color, #dee2e6); border-top-right-radius: 0; border-bottom-right-radius: 0; height: 42px; } .hsm-url-group .btn { flex: 0 0 auto; margin: 0; margin-left: -1px; border-top-left-radius: 0; border-bottom-left-radius: 0; height: 42px; display: flex; align-items: center; justify-content: center; padding: 0 1.25rem; z-index: 2; } [data-bs-theme="dark"] .hsm-url-group .form-control { background: #0f172a; border-color: #334155; color: #e2e8f0; } .hsm-share-grid { display: flex; flex-direction: column; gap: 0.5rem; } .hsm-share-grid .btn { display: flex; align-items: center; justify-content: center; font-size: 0.9rem; padding: 0.6rem; border-color: var(--bs-border-color); width: 100%; } [data-bs-theme="dark"] .hsm-share-grid .btn { color: #e2e8f0; border-color: #475569; } [data-bs-theme="dark"] .hsm-share-grid .btn:hover { background: #334155; border-color: #cbd5e1; } </style> <script> (function(){ const modal = document.getElementById('headingShareModal'); if(!modal) return; const input = modal.querySelector('#headingShareInput'); const copyBtn = modal.querySelector('.hsm-copy'); const twitter = modal.querySelector('#share-twitter'); const linkedin = modal.querySelector('#share-linkedin'); const facebook = modal.querySelector('#share-facebook'); const closeBtn = modal.querySelector('.hsm-close'); let lastFocus=null; let trapBound=false; function buildUrl(id){ return window.location.origin + window.location.pathname + '#' + id; } function isOpen(){ return !modal.hasAttribute('hidden'); } function hydrate(id){ const url=buildUrl(id); input.value=url; const enc=encodeURIComponent(url); const text=encodeURIComponent(document.title); if(twitter) twitter.href=`https://twitter.com/intent/tweet?url=${enc}&text=${text}`; if(linkedin) linkedin.href=`https://www.linkedin.com/sharing/share-offsite/?url=${enc}`; if(facebook) facebook.href=`https://www.facebook.com/sharer/sharer.php?u=${enc}`; } function openModal(id){ lastFocus=document.activeElement; hydrate(id); if(!isOpen()){ modal.removeAttribute('hidden'); } requestAnimationFrame(()=>{ input.focus(); }); trapFocus(); } function closeModal(){ if(!isOpen()) return; modal.setAttribute('hidden',''); if(lastFocus && typeof lastFocus.focus==='function') lastFocus.focus(); } function copyCurrent(){ try{ navigator.clipboard.writeText(input.value).then(()=>feedback(true),()=>fallback()); } catch(e){ fallback(); } } function fallback(){ input.select(); try{ document.execCommand('copy'); feedback(true);}catch(e){ feedback(false);} } function feedback(ok){ if(!copyBtn) return; const icon=copyBtn.querySelector('i'); if(!icon) return; const prev=copyBtn.getAttribute('data-prev')||icon.className; if(!copyBtn.getAttribute('data-prev')) copyBtn.setAttribute('data-prev',prev); icon.className= ok ? 'fa-duotone fa-clipboard-check':'fa-duotone fa-circle-exclamation'; setTimeout(()=>{ icon.className=prev; },1800); } function handleShareClick(e){ e.preventDefault(); const btn=e.currentTarget; const id=btn.getAttribute('data-share-target'); if(id) openModal(id); } function bindShareButtons(){ document.querySelectorAll('.h-share').forEach(btn=>{ if(!btn.dataset.hShareBound){ btn.addEventListener('click', handleShareClick); btn.dataset.hShareBound='1'; } }); } bindShareButtons(); if(document.readyState==='loading'){ document.addEventListener('DOMContentLoaded', bindShareButtons); } else { requestAnimationFrame(bindShareButtons); } document.addEventListener('click', function(e){ const shareBtn=e.target.closest && e.target.closest('.h-share'); if(shareBtn && !shareBtn.dataset.hShareBound){ handleShareClick.call(shareBtn, e); } }, true); document.addEventListener('click', e=>{ if(e.target===modal) closeModal(); if(e.target.closest && e.target.closest('.hsm-close')){ e.preventDefault(); closeModal(); } if(copyBtn && (e.target===copyBtn || (e.target.closest && e.target.closest('.hsm-copy')))) { e.preventDefault(); copyCurrent(); } }); document.addEventListener('keydown', e=>{ if(e.key==='Escape' && isOpen()) closeModal(); }); function trapFocus(){ if(trapBound) return; trapBound=true; modal.addEventListener('keydown', f=>{ if(f.key==='Tab' && isOpen()){ const focusable=[...modal.querySelectorAll('a[href],button,input,textarea,select,[tabindex]:not([tabindex="-1"])')].filter(el=>!el.hasAttribute('disabled')); if(!focusable.length) return; const first=focusable[0]; const last=focusable[focusable.length-1]; if(f.shiftKey && document.activeElement===first){ f.preventDefault(); last.focus(); } else if(!f.shiftKey && document.activeElement===last){ f.preventDefault(); first.focus(); } } }); } if(closeBtn) closeBtn.addEventListener('click', e=>{ e.preventDefault(); closeModal(); }); })(); </script>Data migration is the process of transferring data from one system to another, often involving format transformation, schema mapping, and data quality validation. Geode provides comprehensive tools and strategies for smooth, reliable data migrations. <h3 id="migration-strategies" class="position-relative d-flex align-items-center group"> Migration Strategies <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="migration-strategies" aria-haspopup="dialog" aria-label="Share link: Migration Strategies"> Share link </button> </h3> <h4 id="big-bang-migration" class="position-relative d-flex align-items-center group"> Big Bang Migration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="big-bang-migration" aria-haspopup="dialog" aria-label="Share link: Big Bang Migration"> Share link </button> </h4>Move all data at once: Pros: <ul> <li>Simple and straightforward</li> <li>Immediate cutover</li> <li>No sync period</li> </ul> Cons: <ul> <li>Higher risk</li> <li>Longer downtime</li> <li>Difficult rollback</li> </ul> <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Stop old system # Export all data geode export --source legacy_db --output data.json # Import to Geode geode import --input data.json --graph production # Start new system </code></pre></div> <h4 id="incremental-migration" class="position-relative d-flex align-items-center group"> Incremental Migration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="incremental-migration" aria-haspopup="dialog" aria-label="Share link: Incremental Migration"> Share link </button> </h4>Migrate data in phases: Pros: <ul> <li>Lower risk</li> <li>Phased validation</li> <li>Easier rollback</li> </ul> Cons: <ul> <li>More complex</li> <li>Requires sync</li> <li>Longer timeline</li> </ul> <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Phase 1: Users geode import --input users.json --graph production # Phase 2: Relationships geode import --input relationships.json --graph production # Phase 3: Activity data geode import --input activities.json --graph production </code></pre></div> <h4 id="parallel-run" class="position-relative d-flex align-items-center group"> Parallel Run <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="parallel-run" aria-haspopup="dialog" aria-label="Share link: Parallel Run"> Share link </button> </h4>Run old and new systems simultaneously: Pros: <ul> <li>Lowest risk</li> <li>Validate in production</li> <li>Gradual transition</li> </ul> Cons: <ul> <li>Most complex</li> <li>Highest cost</li> <li>Sync overhead</li> </ul> <h3 id="schema-migration" class="position-relative d-flex align-items-center group"> Schema Migration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="schema-migration" aria-haspopup="dialog" aria-label="Share link: Schema Migration"> Share link </button> </h3> <h4 id="schema-versioning" class="position-relative d-flex align-items-center group"> Schema Versioning <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="schema-versioning" aria-haspopup="dialog" aria-label="Share link: Schema Versioning"> Share link </button> </h4>Track schema changes: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Create schema version table CREATE TABLE schema_versions ( version INTEGER PRIMARY KEY, description STRING, applied_at TIMESTAMP, applied_by STRING ); -- Record migration INSERT INTO schema_versions VALUES ( 1, 'Initial schema', NOW(), CURRENT_USER() ); </code></pre></div> <h4 id="schema-evolution" class="position-relative d-flex align-items-center group"> Schema Evolution <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="schema-evolution" aria-haspopup="dialog" aria-label="Share link: Schema Evolution"> Share link </button> </h4>Handle breaking changes: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Add new property ALTER PROPERTY TYPE Person ADD COLUMN middle_name STRING; -- Populate from existing data UPDATE Person SET middle_name = SPLIT_PART(full_name, ' ', 2) WHERE full_name LIKE '% % %'; -- Remove old property ALTER PROPERTY TYPE Person DROP COLUMN full_name; </code></pre></div> <h3 id="data-transformation" class="position-relative d-flex align-items-center group"> Data Transformation <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-transformation" aria-haspopup="dialog" aria-label="Share link: Data Transformation"> Share link </button> </h3> <h4 id="type-conversion" class="position-relative d-flex align-items-center group"> Type Conversion <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="type-conversion" aria-haspopup="dialog" aria-label="Share link: Type Conversion"> Share link </button> </h4>Convert data types: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- String to date UPDATE Person SET birth_date = TO_DATE(birth_date_string, 'YYYY-MM-DD') WHERE birth_date IS NULL; -- JSON to structured UPDATE Event SET metadata_parsed = JSON_PARSE(metadata_string); </code></pre></div> <h4 id="data-cleansing" class="position-relative d-flex align-items-center group"> Data Cleansing <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-cleansing" aria-haspopup="dialog" aria-label="Share link: Data Cleansing"> Share link </button> </h4>Clean and normalize data: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Trim whitespace UPDATE Person SET email = TRIM(LOWER(email)); -- Standardize phone numbers UPDATE Contact SET phone = REGEXP_REPLACE(phone, '[^0-9]', ''); -- Remove duplicates DELETE FROM Person p1 WHERE EXISTS ( SELECT 1 FROM Person p2 WHERE p1.email = p2.email AND p1.id > p2.id ); </code></pre></div> <h3 id="migration-tools" class="position-relative d-flex align-items-center group"> Migration Tools <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="migration-tools" aria-haspopup="dialog" aria-label="Share link: Migration Tools"> Share link </button> </h3> <h4 id="geode-cli" class="position-relative d-flex align-items-center group"> Geode CLI <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="geode-cli" aria-haspopup="dialog" aria-label="Share link: Geode CLI"> Share link </button> </h4>Command-line migration tools: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Export from SQL database geode export \ --source postgresql://localhost/mydb \ --output backup.json \ --format json # Import to Geode geode import \ --input backup.json \ --graph production \ --batch-size 1000 \ --parallel 4 # Verify migration geode validate \ --graph production \ --check-constraints \ --check-references </code></pre></div> <h4 id="python-migration-script" class="position-relative d-flex align-items-center group"> Python Migration Script <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="python-migration-script" aria-haspopup="dialog" aria-label="Share link: Python Migration Script"> Share link </button> </h4>Custom migration logic: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">from geode_client import Client import asyncio async def migrate_users(source_conn, geode_client): """Migrate users from SQL to Geode""" # Extract from source users = await source_conn.fetch("SELECT * FROM users") batch = [] for user in users: # Transform node = { "labels": ["User"], "properties": { "id": user['id'], "name": user['name'], "email": user['email'].lower(), "created_at": user['created_at'].isoformat() } } batch.append(node) # Load in batches if len(batch) >= 1000: await geode_client.create_nodes_batch(batch) batch = [] # Load remaining if batch: await geode_client.create_nodes_batch(batch) async def main(): source = await connect_source() geode = Client("localhost", 3141) await migrate_users(source, geode) await migrate_relationships(source, geode) await validate_migration(source, geode) asyncio.run(main()) </code></pre></div> <h3 id="validation" class="position-relative d-flex align-items-center group"> Validation <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="validation" aria-haspopup="dialog" aria-label="Share link: Validation"> Share link </button> </h3> <h4 id="data-quality-checks" class="position-relative d-flex align-items-center group"> Data Quality Checks <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-quality-checks" aria-haspopup="dialog" aria-label="Share link: Data Quality Checks"> Share link </button> </h4>Verify migration completeness: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Count comparison SELECT 'source' AS system, COUNT(*) AS user_count FROM source_users UNION ALL SELECT 'target' AS system, COUNT(*) AS user_count FROM Person; -- Sample verification SELECT * FROM Person p WHERE NOT EXISTS ( SELECT 1 FROM source_users s WHERE s.id = p.id ); </code></pre></div> <h4 id="referential-integrity" class="position-relative d-flex align-items-center group"> Referential Integrity <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="referential-integrity" aria-haspopup="dialog" aria-label="Share link: Referential Integrity"> Share link </button> </h4>Validate relationships: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Check for orphaned edges SELECT e.* FROM FRIEND e WHERE NOT EXISTS ( SELECT 1 FROM Person p WHERE p.id = e.from_id ) OR NOT EXISTS ( SELECT 1 FROM Person p WHERE p.id = e.to_id ); </code></pre></div> <h3 id="best-practices" class="position-relative d-flex align-items-center group"> Best Practices <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="best-practices" aria-haspopup="dialog" aria-label="Share link: Best Practices"> Share link </button> </h3> <h4 id="planning" class="position-relative d-flex align-items-center group"> Planning <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="planning" aria-haspopup="dialog" aria-label="Share link: Planning"> Share link </button> </h4><ul> <li>Document current state</li> <li>Define success criteria</li> <li>Create rollback plan</li> <li>Test on copy first</li> <li>Schedule during low-traffic</li> </ul> <h4 id="execution" class="position-relative d-flex align-items-center group"> Execution <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="execution" aria-haspopup="dialog" aria-label="Share link: Execution"> Share link </button> </h4><ul> <li>Use transactions</li> <li>Batch operations</li> <li>Monitor progress</li> <li>Validate continuously</li> <li>Keep audit trail</li> </ul> <h4 id="post-migration" class="position-relative d-flex align-items-center group"> Post-Migration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="post-migration" aria-haspopup="dialog" aria-label="Share link: Post-Migration"> Share link </button> </h4><ul> <li>Verify data quality</li> <li>Update documentation</li> <li>Monitor performance</li> <li>Gather feedback</li> <li>Archive old data</li> </ul> <h3 id="related-topics" class="position-relative d-flex align-items-center group"> Related Topics <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="related-topics" aria-haspopup="dialog" aria-label="Share link: Related Topics"> Share link </button> </h3><ul> <li><a href="/tags/import" >Import</a> - Data import operations</li> <li><a href="/tags/export" >Export</a> - Data export operations</li> <li><a href="/tags/etl" >ETL</a> - Extract Transform Load pipelines</li> </ul> <h3 id="advanced-migration-patterns" class="position-relative d-flex align-items-center group"> Advanced Migration Patterns <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="advanced-migration-patterns" aria-haspopup="dialog" aria-label="Share link: Advanced Migration Patterns"> Share link </button> </h3> <h4 id="zero-downtime-migration-strategy" class="position-relative d-flex align-items-center group"> Zero-Downtime Migration Strategy <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="zero-downtime-migration-strategy" aria-haspopup="dialog" aria-label="Share link: Zero-Downtime Migration Strategy"> Share link </button> </h4>Deploy changes without service interruption: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">from geode_client import Client import asyncio class ZeroDowntimeMigration: """Manage migrations with zero downtime using dual-write pattern.""" def __init__(self, old_db, new_db): self.old_db = old_db self.new_db = new_db self.migration_complete = False async def phase1_dual_write(self): """Phase 1: Write to both old and new systems.""" print("Phase 1: Starting dual-write mode") # All new writes go to both systems async def write_user(user_data): # Write to old system await self.old_db.execute(""" INSERT INTO users_table VALUES ($id, $name, $email) """, user_data) # Write to new system (Geode) await self.new_db.execute(""" CREATE (u:User { id: $id, name: $name, email: $email, migrated_at: datetime() }) """, user_data) return write_user async def phase2_backfill(self, batch_size=1000): """Phase 2: Backfill historical data.""" print("Phase 2: Backfilling historical data") offset = 0 while True: # Read batch from old system old_users = await self.old_db.fetch(f""" SELECT id, name, email, created_at FROM users_table ORDER BY id LIMIT {batch_size} OFFSET {offset} """) if not old_users: break # Write to new system async with self.new_db.transaction(): await self.new_db.execute(""" UNWIND $users AS user MERGE (u:User {id: user.id}) ON CREATE SET u.name = user.name, u.email = user.email, u.created_at = user.created_at, u.backfilled = true """, {'users': old_users}) offset += batch_size print(f"Backfilled {offset} users") async def phase3_verify(self): """Phase 3: Verify data consistency.""" print("Phase 3: Verifying data consistency") # Count records old_count = await self.old_db.fetch_val("SELECT COUNT(*) FROM users_table") new_count = await self.new_db.execute("MATCH (u:User) RETURN count(u) AS count") new_count = (new_count.rows[0] if new_count.rows else None)['count'] if old_count != new_count: raise MigrationError(f"Count mismatch: {old_count} vs {new_count}") # Sample verification sample_ids = await self.old_db.fetch("SELECT id FROM users_table ORDER BY RANDOM() LIMIT 1000") for sample in sample_ids: old_user = await self.old_db.fetch_one(f"SELECT * FROM users_table WHERE id = {sample['id']}") new_user = await self.new_db.execute("MATCH (u:User {id: $id}) RETURN u", {'id': sample['id']}) new_user = new_user.rows[0] if new_user.rows else None if not new_user or old_user['name'] != new_user['u']['name']: raise MigrationError(f"Data mismatch for user {sample['id']}") print("Verification complete") async def phase4_cutover(self): """Phase 4: Switch reads to new system.""" print("Phase 4: Cutting over reads to new system") self.migration_complete = True # Usage async def run_zero_downtime_migration(): old_postgres = PostgresClient("old-db:5432") new_geode = Client("new-geode:3141") migration = ZeroDowntimeMigration(old_postgres, new_geode) # Run migration phases await migration.phase1_dual_write() await migration.phase2_backfill(batch_size=10000) await migration.phase3_verify() await migration.phase4_cutover() print("Migration complete!") asyncio.run(run_zero_downtime_migration()) </code></pre></div> <h4 id="schema-transformation-patterns" class="position-relative d-flex align-items-center group"> Schema Transformation Patterns <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="schema-transformation-patterns" aria-haspopup="dialog" aria-label="Share link: Schema Transformation Patterns"> Share link </button> </h4>Transform relational schema to graph model: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">class RelationalToGraphTransformer: """Transform relational database schema to graph model.""" def __init__(self, source_db, target_graph): self.source = source_db self.target = target_graph async def migrate_entities(self, table_name, label): """Migrate table rows to graph nodes.""" print(f"Migrating {table_name} → :{label}") # Read table schema columns = await self.source.get_columns(table_name) # Migrate data in batches offset = 0 batch_size = 5000 while True: rows = await self.source.fetch(f""" SELECT * FROM {table_name} ORDER BY id LIMIT {batch_size} OFFSET {offset} """) if not rows: break # Transform and insert nodes = [ {col: row[col] for col in columns} for row in rows ] await self.target.execute(f""" UNWIND $nodes AS node CREATE (n:{label}) SET n = node """, {'nodes': nodes}) offset += batch_size print(f" Migrated {offset} {label} nodes") async def migrate_foreign_keys(self, fk_constraint): """Transform foreign keys to relationships.""" table = fk_constraint['table'] ref_table = fk_constraint['referenced_table'] fk_column = fk_constraint['column'] ref_column = fk_constraint['referenced_column'] rel_type = self.infer_relationship_type(table, ref_table) print(f"Creating {rel_type} relationships: {table} → {ref_table}") await self.target.execute(f""" MATCH (a:{table.title()}) MATCH (b:{ref_table.title()} {{id: a.{fk_column}}}) CREATE (a)-[:{rel_type}]->(b) """) def infer_relationship_type(self, from_table, to_table): """Infer relationship type from table names.""" # Simple heuristic - customize for your schema return f"HAS_{to_table.upper()}" async def migrate_join_tables(self, join_table): """Transform many-to-many join tables to relationships.""" left_col = join_table['left_column'] right_col = join_table['right_column'] left_entity = join_table['left_entity'] right_entity = join_table['right_entity'] rel_type = f"{left_entity}_TO_{right_entity}".upper() print(f"Migrating join table {join_table['name']} → {rel_type}") rows = await self.source.fetch(f""" SELECT * FROM {join_table['name']} """) for row in rows: await self.target.execute(f""" MATCH (a:{left_entity} {{id: $left_id}}) MATCH (b:{right_entity} {{id: $right_id}}) CREATE (a)-[r:{rel_type}]->(b) SET r = $properties """, { 'left_id': row[left_col], 'right_id': row[right_col], 'properties': {k: v for k, v in row.items() if k not in [left_col, right_col]} }) # Example usage async def migrate_ecommerce_schema(): transformer = RelationalToGraphTransformer(postgres, geode) # Migrate entities await transformer.migrate_entities('users', 'User') await transformer.migrate_entities('products', 'Product') await transformer.migrate_entities('orders', 'Order') # Migrate foreign keys await transformer.migrate_foreign_keys({ 'table': 'orders', 'referenced_table': 'users', 'column': 'user_id', 'referenced_column': 'id' }) # Migrate join tables await transformer.migrate_join_tables({ 'name': 'order_items', 'left_column': 'order_id', 'right_column': 'product_id', 'left_entity': 'Order', 'right_entity': 'Product' }) </code></pre></div> <h4 id="data-quality-validation" class="position-relative d-flex align-items-center group"> Data Quality Validation <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-quality-validation" aria-haspopup="dialog" aria-label="Share link: Data Quality Validation"> Share link </button> </h4>Comprehensive validation during migration: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">class MigrationValidator: """Validate data quality during and after migration.""" def __init__(self, source_db, target_graph): self.source = source_db self.target = target_graph self.issues = [] async def validate_counts(self, table_name, label): """Verify row/node counts match.""" source_count = await self.source.fetch_val(f"SELECT COUNT(*) FROM {table_name}") result = await self.target.execute(f"MATCH (n:{label}) RETURN count(n) AS count") target_count = (result.rows[0] if result.rows else None)['count'] if source_count != target_count: self.issues.append({ 'type': 'count_mismatch', 'entity': label, 'source': source_count, 'target': target_count }) return source_count == target_count async def validate_referential_integrity(self): """Check for orphaned relationships.""" result = await self.target.execute(""" MATCH ()-[r]->() WHERE NOT EXISTS { MATCH (start)-[r]->(end) WHERE id(start) IS NOT NULL AND id(end) IS NOT NULL } RETURN count(r) AS orphaned_relationships """) orphaned = (result.rows[0] if result.rows else None)['orphaned_relationships'] if orphaned > 0: self.issues.append({ 'type': 'orphaned_relationships', 'count': orphaned }) return orphaned == 0 async def validate_data_types(self, label, property_name, expected_type): """Verify property data types.""" result = await self.target.execute(f""" MATCH (n:{label}) WHERE n.{property_name} IS NOT NULL WITH n.{property_name} AS value, valueType(n.{property_name}) AS actual_type WHERE actual_type <> $expected_type RETURN count(*) AS type_mismatches """, {'expected_type': expected_type}) mismatches = (result.rows[0] if result.rows else None)['type_mismatches'] if mismatches > 0: self.issues.append({ 'type': 'data_type_mismatch', 'entity': label, 'property': property_name, 'count': mismatches }) return mismatches == 0 async def validate_unique_constraints(self, label, property_name): """Check for duplicate values in unique properties.""" result = await self.target.execute(f""" MATCH (n:{label}) WITH n.{property_name} AS value, count(n) AS occurrences WHERE occurrences > 1 RETURN count(*) AS duplicates """) duplicates = (result.rows[0] if result.rows else None)['duplicates'] if duplicates > 0: self.issues.append({ 'type': 'unique_constraint_violation', 'entity': label, 'property': property_name, 'count': duplicates }) return duplicates == 0 def generate_report(self): """Generate validation report.""" if not self.issues: return "✓ All validations passed" report = "Migration Validation Issues:\n" for issue in self.issues: report += f"\n• {issue['type']}:\n" for key, value in issue.items(): if key != 'type': report += f" - {key}: {value}\n" return report </code></pre></div> <h4 id="incremental-migration-with-change-data-capture" class="position-relative d-flex align-items-center group"> Incremental Migration with Change Data Capture <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="incremental-migration-with-change-data-capture" aria-haspopup="dialog" aria-label="Share link: Incremental Migration with Change Data Capture"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">class CDCMigration: """Migrate data incrementally using change data capture.""" def __init__(self, source_db, target_graph): self.source = source_db self.target = target_graph self.last_sync_timestamp = None async def capture_changes(self, since_timestamp=None): """Capture changes from source database.""" timestamp = since_timestamp or self.last_sync_timestamp or '1970-01-01' changes = await self.source.fetch(""" SELECT table_name, operation, record_id, new_data, changed_at FROM change_log WHERE changed_at > $timestamp ORDER BY changed_at """, {'timestamp': timestamp}) return changes async def apply_changes(self, changes): """Apply changes to target graph database.""" for change in changes: table = change['table_name'] operation = change['operation'] data = change['new_data'] if operation == 'INSERT' or operation == 'UPDATE': await self.target.execute(f""" MERGE (n:{table.title()} {{id: $id}}) SET n = $data, n.synced_at = datetime() """, {'id': change['record_id'], 'data': data}) elif operation == 'DELETE': await self.target.execute(f""" MATCH (n:{table.title()} {{id: $id}}) DETACH DELETE n """, {'id': change['record_id']}) if changes: self.last_sync_timestamp = changes[-1]['changed_at'] async def continuous_sync(self, interval_seconds=60): """Continuously sync changes.""" print("Starting continuous sync...") while True: try: changes = await self.capture_changes() if changes: await self.apply_changes(changes) print(f"Synced {len(changes)} changes") await asyncio.sleep(interval_seconds) except Exception as e: print(f"Sync error: {e}") await asyncio.sleep(interval_seconds) </code></pre></div> <h4 id="migration-rollback-strategy" class="position-relative d-flex align-items-center group"> Migration Rollback Strategy <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="migration-rollback-strategy" aria-haspopup="dialog" aria-label="Share link: Migration Rollback Strategy"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">class MigrationRollback: """Provide rollback capability for failed migrations.""" def __init__(self, target_graph): self.target = target_graph self.migration_id = None self.checkpoints = [] async def create_checkpoint(self, name): """Create a recovery checkpoint.""" checkpoint = { 'name': name, 'timestamp': datetime.now(), 'node_count': await self.get_node_count(), 'relationship_count': await self.get_relationship_count() } self.checkpoints.append(checkpoint) print(f"Checkpoint created: {name}") return checkpoint async def get_node_count(self): """Get current node count.""" result = await self.target.execute("MATCH (n) RETURN count(n) AS count") return (result.rows[0] if result.rows else None)['count'] async def get_relationship_count(self): """Get current relationship count.""" result = await self.target.execute("MATCH ()-[r]->() RETURN count(r) AS count") return (result.rows[0] if result.rows else None)['count'] async def rollback_to_checkpoint(self, checkpoint_name): """Rollback to a specific checkpoint.""" checkpoint = next((c for c in self.checkpoints if c['name'] == checkpoint_name), None) if not checkpoint: raise ValueError(f"Checkpoint not found: {checkpoint_name}") print(f"Rolling back to checkpoint: {checkpoint_name}") # Delete nodes created after checkpoint await self.target.execute(""" MATCH (n) WHERE n.migrated_at > $checkpoint_time DETACH DELETE n """, {'checkpoint_time': checkpoint['timestamp']}) print("Rollback complete") ## Performance Optimization ### Parallel Migration Processing ```python class ParallelMigrator: """Migrate data using parallel workers.""" def __init__(self, source_db, target_graph, num_workers=4): self.source = source_db self.target = target_graph self.num_workers = num_workers async def migrate_table_parallel(self, table_name, label): """Migrate table using parallel workers.""" # Get total row count total_rows = await self.source.fetch_val(f"SELECT COUNT(*) FROM {table_name}") chunk_size = total_rows // self.num_workers + 1 # Create worker tasks tasks = [] for worker_id in range(self.num_workers): offset = worker_id * chunk_size task = self.migrate_chunk(table_name, label, offset, chunk_size, worker_id) tasks.append(task) # Execute in parallel results = await asyncio.gather(*tasks) total_migrated = sum(results) print(f"Migrated {total_migrated} rows from {table_name}") async def migrate_chunk(self, table_name, label, offset, limit, worker_id): """Migrate a chunk of data (worker task).""" rows = await self.source.fetch(f""" SELECT * FROM {table_name} ORDER BY id LIMIT {limit} OFFSET {offset} """) if not rows: return 0 # Create connection for this worker worker_client = Client(host="localhost", port=3141) async with worker_client.connection() as conn: await conn.begin() try: await conn.execute(f""" UNWIND $rows AS row CREATE (n:{label}) SET n = row """, {'rows': rows}) await conn.commit() print(f"Worker {worker_id}: Migrated {len(rows)} rows") return len(rows) except Exception: await conn.rollback() raise </code></pre></div> <h3 id="learn-more" class="position-relative d-flex align-items-center group"> Learn More <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="learn-more" aria-haspopup="dialog" aria-label="Share link: Learn More"> Share link </button> </h3><ul> <li><a href="https://www.liquibase.org/best-practices.html" aria-label="Database Migration Best Practices – opens in new window" target="_blank" rel="noopener noreferrer" >Database Migration Best Practices ↗ </a> </li> <li><a href="https://stripe.com/blog/online-migrations" aria-label="Zero-Downtime Migrations – opens in new window" target="_blank" rel="noopener noreferrer" >Zero-Downtime Migrations ↗ </a> </li> </ul>

Popular

Related Articles

Schema Migration Guide

Data Import Guide

Guides

Migration Guide

Migrating from Neo4j to Geode

Migrating from PostgreSQL to Geode

Migrating from MongoDB to Geode