Data Import - Loading Data into Geode | Tags

<h2 id="data-import" class="position-relative d-flex align-items-center group"> Data Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-import" aria-haspopup="dialog" aria-label="Share link: Data Import"> Share link </button> </h2><div id="headingShareModal" class="heading-share-modal" role="dialog" aria-modal="true" aria-labelledby="headingShareTitle" hidden> <div class="hsm-dialog" role="document"> <div class="hsm-header"> <h2 id="headingShareTitle" class="h6 mb-0 fw-bold">Share this section</h2> <button type="button" class="hsm-close" aria-label="Close"> </button> </div> <div class="hsm-body"> <label for="headingShareInput" class="form-label small text-muted mb-1 text-uppercase fw-bold" style="font-size: 0.7rem; letter-spacing: 0.5px;">Permalink</label> <div class="input-group mb-4 hsm-url-group"> <input id="headingShareInput" type="text" class="form-control font-monospace" readonly aria-readonly="true" style="font-size: 0.85rem;" /> <button class="btn btn-primary hsm-copy" type="button" aria-label="Copy" title="Copy"> </button> </div> <div class="small fw-bold mb-2 text-muted text-uppercase" style="font-size: 0.7rem; letter-spacing: 0.5px;">Share via</div> <div class="hsm-share-grid"> <a id="share-twitter" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Twitter </a> <a id="share-linkedin" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> LinkedIn </a> <a id="share-facebook" class="btn btn-outline-secondary w-100" target="_blank" rel="noopener noreferrer"> Facebook </a> </div> </div> </div> </div> <style> .heading-share-modal { position: fixed; inset: 0; display: flex; justify-content: center; align-items: center; background: rgba(0, 0, 0, 0.6); z-index: 1050; padding: 1rem; backdrop-filter: blur(4px); -webkit-backdrop-filter: blur(4px); } .heading-share-modal[hidden] { display: none !important; } .hsm-dialog { max-width: 420px; width: 100%; background: var(--bs-body-bg, #fff); color: var(--bs-body-color, #212529); border: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); border-radius: 1rem; box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.25); overflow: hidden; animation: hsm-fade-in 0.2s ease-out; } @keyframes hsm-fade-in { from { opacity: 0; transform: scale(0.95); } to { opacity: 1; transform: scale(1); } } [data-bs-theme="dark"] .hsm-dialog { background: #1e293b; border-color: rgba(255,255,255,0.1); color: #f8f9fa; } .hsm-header { display: flex; justify-content: space-between; align-items: center; padding: 1rem 1.5rem; border-bottom: 1px solid var(--bs-border-color, rgba(0,0,0,0.1)); background: rgba(0,0,0,0.02); } [data-bs-theme="dark"] .hsm-header { background: rgba(255,255,255,0.02); border-color: rgba(255,255,255,0.1); } .hsm-close { background: transparent; border: none; color: inherit; opacity: 0.5; padding: 0.25rem 0.5rem; border-radius: 0.25rem; font-size: 1.2rem; line-height: 1; transition: opacity 0.2s; } .hsm-close:hover { opacity: 1; } .hsm-body { padding: 1.5rem; } .hsm-url-group { display: flex !important; align-items: stretch; } .hsm-url-group .form-control { flex: 1; min-width: 0; margin: 0; background: var(--bs-secondary-bg, #f8f9fa); border-color: var(--bs-border-color, #dee2e6); border-top-right-radius: 0; border-bottom-right-radius: 0; height: 42px; } .hsm-url-group .btn { flex: 0 0 auto; margin: 0; margin-left: -1px; border-top-left-radius: 0; border-bottom-left-radius: 0; height: 42px; display: flex; align-items: center; justify-content: center; padding: 0 1.25rem; z-index: 2; } [data-bs-theme="dark"] .hsm-url-group .form-control { background: #0f172a; border-color: #334155; color: #e2e8f0; } .hsm-share-grid { display: flex; flex-direction: column; gap: 0.5rem; } .hsm-share-grid .btn { display: flex; align-items: center; justify-content: center; font-size: 0.9rem; padding: 0.6rem; border-color: var(--bs-border-color); width: 100%; } [data-bs-theme="dark"] .hsm-share-grid .btn { color: #e2e8f0; border-color: #475569; } [data-bs-theme="dark"] .hsm-share-grid .btn:hover { background: #334155; border-color: #cbd5e1; } </style> <script> (function(){ const modal = document.getElementById('headingShareModal'); if(!modal) return; const input = modal.querySelector('#headingShareInput'); const copyBtn = modal.querySelector('.hsm-copy'); const twitter = modal.querySelector('#share-twitter'); const linkedin = modal.querySelector('#share-linkedin'); const facebook = modal.querySelector('#share-facebook'); const closeBtn = modal.querySelector('.hsm-close'); let lastFocus=null; let trapBound=false; function buildUrl(id){ return window.location.origin + window.location.pathname + '#' + id; } function isOpen(){ return !modal.hasAttribute('hidden'); } function hydrate(id){ const url=buildUrl(id); input.value=url; const enc=encodeURIComponent(url); const text=encodeURIComponent(document.title); if(twitter) twitter.href=`https://twitter.com/intent/tweet?url=${enc}&text=${text}`; if(linkedin) linkedin.href=`https://www.linkedin.com/sharing/share-offsite/?url=${enc}`; if(facebook) facebook.href=`https://www.facebook.com/sharer/sharer.php?u=${enc}`; } function openModal(id){ lastFocus=document.activeElement; hydrate(id); if(!isOpen()){ modal.removeAttribute('hidden'); } requestAnimationFrame(()=>{ input.focus(); }); trapFocus(); } function closeModal(){ if(!isOpen()) return; modal.setAttribute('hidden',''); if(lastFocus && typeof lastFocus.focus==='function') lastFocus.focus(); } function copyCurrent(){ try{ navigator.clipboard.writeText(input.value).then(()=>feedback(true),()=>fallback()); } catch(e){ fallback(); } } function fallback(){ input.select(); try{ document.execCommand('copy'); feedback(true);}catch(e){ feedback(false);} } function feedback(ok){ if(!copyBtn) return; const icon=copyBtn.querySelector('i'); if(!icon) return; const prev=copyBtn.getAttribute('data-prev')||icon.className; if(!copyBtn.getAttribute('data-prev')) copyBtn.setAttribute('data-prev',prev); icon.className= ok ? 'fa-duotone fa-clipboard-check':'fa-duotone fa-circle-exclamation'; setTimeout(()=>{ icon.className=prev; },1800); } function handleShareClick(e){ e.preventDefault(); const btn=e.currentTarget; const id=btn.getAttribute('data-share-target'); if(id) openModal(id); } function bindShareButtons(){ document.querySelectorAll('.h-share').forEach(btn=>{ if(!btn.dataset.hShareBound){ btn.addEventListener('click', handleShareClick); btn.dataset.hShareBound='1'; } }); } bindShareButtons(); if(document.readyState==='loading'){ document.addEventListener('DOMContentLoaded', bindShareButtons); } else { requestAnimationFrame(bindShareButtons); } document.addEventListener('click', function(e){ const shareBtn=e.target.closest && e.target.closest('.h-share'); if(shareBtn && !shareBtn.dataset.hShareBound){ handleShareClick.call(shareBtn, e); } }, true); document.addEventListener('click', e=>{ if(e.target===modal) closeModal(); if(e.target.closest && e.target.closest('.hsm-close')){ e.preventDefault(); closeModal(); } if(copyBtn && (e.target===copyBtn || (e.target.closest && e.target.closest('.hsm-copy')))) { e.preventDefault(); copyCurrent(); } }); document.addEventListener('keydown', e=>{ if(e.key==='Escape' && isOpen()) closeModal(); }); function trapFocus(){ if(trapBound) return; trapBound=true; modal.addEventListener('keydown', f=>{ if(f.key==='Tab' && isOpen()){ const focusable=[...modal.querySelectorAll('a[href],button,input,textarea,select,[tabindex]:not([tabindex="-1"])')].filter(el=>!el.hasAttribute('disabled')); if(!focusable.length) return; const first=focusable[0]; const last=focusable[focusable.length-1]; if(f.shiftKey && document.activeElement===first){ f.preventDefault(); last.focus(); } else if(!f.shiftKey && document.activeElement===last){ f.preventDefault(); first.focus(); } } }); } if(closeBtn) closeBtn.addEventListener('click', e=>{ e.preventDefault(); closeModal(); }); })(); </script>Data import loads external data into Geode from various sources and formats. Efficient import strategies ensure fast, reliable data loading while maintaining data integrity, handling errors gracefully, and optimizing for system performance during bulk operations. <h3 id="import-formats" class="position-relative d-flex align-items-center group"> Import Formats <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="import-formats" aria-haspopup="dialog" aria-label="Share link: Import Formats"> Share link </button> </h3> <h4 id="json-import" class="position-relative d-flex align-items-center group"> JSON Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="json-import" aria-haspopup="dialog" aria-label="Share link: JSON Import"> Share link </button> </h4>Import graph structure from JSON files: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Import JSON file geode import --format json --input backup.json --graph production # Import with validation geode import --format json --input data.json --validate --graph production # Import JSON lines (streaming format) geode import --format jsonl --input large_dataset.jsonl --stream --graph production </code></pre></div>JSON Format: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-json" data-lang="json">{ "nodes": [ { "labels": ["Person"], "properties": { "name": "Alice Johnson", "age": 30, "email": "[email protected]" } }, { "labels": ["Company"], "properties": { "name": "Acme Corp", "industry": "Technology" } } ], "relationships": [ { "type": "WORKS_FOR", "start_node_id": "person_1", "end_node_id": "company_1", "properties": { "since": "2020-01-15", "role": "Engineer" } } ] } </code></pre></div> <h4 id="csv-import" class="position-relative d-flex align-items-center group"> CSV Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="csv-import" aria-haspopup="dialog" aria-label="Share link: CSV Import"> Share link </button> </h4>Import tabular data from CSV files: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Import CSV with headers geode import --format csv --input users.csv --has-header --graph production # Import without headers (specify column mapping) geode import --format csv --input data.csv \ --columns "id,name,email,age" \ --node-type Person \ --graph production # Custom delimiter (TSV) geode import --format csv --input data.tsv --delimiter "\t" --graph production </code></pre></div>CSV Format: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">id,name,email,age,city 1,Alice Johnson,[email protected],30,San Francisco 2,Bob Smith,[email protected],25,New York 3,Carol White,[email protected],35,Seattle </code></pre></div> <h4 id="graphml-import" class="position-relative d-flex align-items-center group"> GraphML Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="graphml-import" aria-haspopup="dialog" aria-label="Share link: GraphML Import"> Share link </button> </h4>Import from graph visualization tools: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Import GraphML file geode import --format graphml --input network.graphml --graph social # Import with attribute mapping geode import --format graphml --input graph.graphml \ --map-attributes "id:nodeId,label:nodeLabel" \ --graph production </code></pre></div> <h3 id="gql-load-statements" class="position-relative d-flex align-items-center group"> GQL LOAD Statements <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="gql-load-statements" aria-haspopup="dialog" aria-label="Share link: GQL LOAD Statements"> Share link </button> </h3>Import data using GQL directly: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Load CSV data LOAD CSV FROM '/data/users.csv' WITH HEADERS AS row CREATE (p:Person { id: row.id, name: row.name, email: row.email, age: toInteger(row.age) }); -- Load with filtering LOAD CSV FROM '/data/products.csv' WITH HEADERS AS row WHERE row.price IS NOT NULL AND toDecimal(row.price) > 0 CREATE (p:Product { id: row.id, name: row.name, price: toDecimal(row.price), category: row.category }); -- Load relationships LOAD CSV FROM '/data/connections.csv' WITH HEADERS AS row MATCH (a:Person {id: row.person_a_id}) MATCH (b:Person {id: row.person_b_id}) CREATE (a)-[:KNOWS {since: date(row.since)}]->(b); -- Load JSON LOAD JSON FROM '/data/users.json' AS item CREATE (p:Person { id: item.id, name: item.name, email: item.email, age: item.age }); </code></pre></div> <h3 id="batch-import" class="position-relative d-flex align-items-center group"> Batch Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="batch-import" aria-haspopup="dialog" aria-label="Share link: Batch Import"> Share link </button> </h3> <h4 id="large-dataset-import" class="position-relative d-flex align-items-center group"> Large Dataset Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="large-dataset-import" aria-haspopup="dialog" aria-label="Share link: Large Dataset Import"> Share link </button> </h4>Import millions of records efficiently: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python"># Python batch import import asyncio from geode_client import Client import csv async def batch_import(filename, batch_size=1000): client = Client(host="localhost", port=3141) async with client.connection() as conn: batch = [] total_imported = 0 with open(filename, 'r') as f: reader = csv.DictReader(f) for row in reader: batch.append(row) if len(batch) >= batch_size: # Import batch in transaction async with client.connection() as tx: await tx.begin() for record in batch: await tx.execute(""" CREATE (p:Person { id: $id, name: $name, email: $email, age: $age }) """, record) await tx.commit() total_imported += len(batch) print(f"Imported {total_imported} records...") batch = [] # Import remaining records if batch: async with client.connection() as tx: await tx.begin() for record in batch: await tx.execute(""" CREATE (p:Person { id: $id, name: $name, email: $email, age: $age }) """, record) await tx.commit() total_imported += len(batch) print(f"Import complete: {total_imported} total records") asyncio.run(batch_import('users.csv')) </code></pre></div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-go" data-lang="go">// Go batch import package main import ( "database/sql" "encoding/csv" "fmt" "os" "strconv" _ "geodedb.com/geode" ) func batchImport(filename string, batchSize int) error { db, err := sql.Open("geode", "quic://localhost:3141") if err != nil { return err } defer db.Close() file, err := os.Open(filename) if err != nil { return err } defer file.Close() reader := csv.NewReader(file) records, err := reader.ReadAll() if err != nil { return err } totalImported := 0 for i := 0; i < len(records); i += batchSize { end := i + batchSize if end > len(records) { end = len(records) } batch := records[i:end] // Start transaction tx, err := db.Begin() if err != nil { return err } stmt, err := tx.Prepare("CREATE (p:Person {id: $1, name: $2, email: $3, age: $4})") if err != nil { tx.Rollback() return err } for _, record := range batch { age, _ := strconv.Atoi(record[3]) _, err = stmt.Exec(record[0], record[1], record[2], age) if err != nil { tx.Rollback() return err } } if err := tx.Commit(); err != nil { return err } totalImported += len(batch) fmt.Printf("Imported %d records...\n", totalImported) } fmt.Printf("Import complete: %d total records\n", totalImported) return nil } </code></pre></div> <h4 id="parallel-import" class="position-relative d-flex align-items-center group"> Parallel Import <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="parallel-import" aria-haspopup="dialog" aria-label="Share link: Parallel Import"> Share link </button> </h4>Speed up imports with parallel workers: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Split large file into chunks split -l 10000 large_dataset.csv chunk_ # Import chunks in parallel for chunk in chunk_*; do geode import --input "$chunk" --format csv --graph production & done wait echo "Parallel import complete" </code></pre></div><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python"># Python parallel import with multiprocessing from multiprocessing import Pool import asyncio from geode_client import Client async def import_chunk(chunk_file): client = Client(host="localhost", port=3141) async with client.connection() as conn: # Import logic for chunk # ... (similar to batch import) pass def import_chunk_sync(chunk_file): asyncio.run(import_chunk(chunk_file)) # Main chunk_files = ['chunk_00', 'chunk_01', 'chunk_02', 'chunk_03'] with Pool(processes=4) as pool: pool.map(import_chunk_sync, chunk_files) </code></pre></div> <h3 id="error-handling" class="position-relative d-flex align-items-center group"> Error Handling <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="error-handling" aria-haspopup="dialog" aria-label="Share link: Error Handling"> Share link </button> </h3> <h4 id="validation" class="position-relative d-flex align-items-center group"> Validation <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="validation" aria-haspopup="dialog" aria-label="Share link: Validation"> Share link </button> </h4>Validate data before import: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Validate email format LOAD CSV FROM '/data/users.csv' WITH HEADERS AS row WHERE row.email LIKE '%@%' CREATE (p:Person { id: row.id, name: row.name, email: row.email }); -- Skip invalid records, log errors LOAD CSV FROM '/data/products.csv' WITH HEADERS AS row WHERE row.price IS NOT NULL AND TRY_CAST(row.price AS DECIMAL) IS NOT NULL CREATE (p:Product { id: row.id, name: row.name, price: CAST(row.price AS DECIMAL) }) ON ERROR CONTINUE LOG TO '/logs/import_errors.log'; </code></pre></div> <h4 id="duplicate-handling" class="position-relative d-flex align-items-center group"> Duplicate Handling <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="duplicate-handling" aria-haspopup="dialog" aria-label="Share link: Duplicate Handling"> Share link </button> </h4>Handle duplicate data: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Use MERGE for upsert behavior LOAD CSV FROM '/data/users.csv' WITH HEADERS AS row MERGE (p:Person {id: row.id}) ON CREATE SET p.name = row.name, p.email = row.email, p.created_at = NOW() ON MATCH SET p.name = row.name, p.email = row.email, p.updated_at = NOW(); -- Skip duplicates LOAD CSV FROM '/data/products.csv' WITH HEADERS AS row WHERE NOT EXISTS { MATCH (p:Product {id: row.id}) } CREATE (p:Product { id: row.id, name: row.name, price: toDecimal(row.price) }); </code></pre></div> <h3 id="data-transformation" class="position-relative d-flex align-items-center group"> Data Transformation <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-transformation" aria-haspopup="dialog" aria-label="Share link: Data Transformation"> Share link </button> </h3> <h4 id="type-conversion" class="position-relative d-flex align-items-center group"> Type Conversion <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="type-conversion" aria-haspopup="dialog" aria-label="Share link: Type Conversion"> Share link </button> </h4>Transform data during import: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Convert string dates to DATE type LOAD CSV FROM '/data/events.csv' WITH HEADERS AS row CREATE (e:Event { id: row.id, name: row.name, event_date: DATE(row.event_date), timestamp: TIMESTAMP(row.timestamp), attendees: toInteger(row.attendees) }); -- Parse JSON strings LOAD CSV FROM '/data/metadata.csv' WITH HEADERS AS row CREATE (p:Product { id: row.id, name: row.name, tags: parseJSON(row.tags), -- "['tag1','tag2']" -> LIST metadata: parseJSON(row.metadata) -- "{\"key\":\"value\"}" -> MAP }); </code></pre></div> <h4 id="data-enrichment" class="position-relative d-flex align-items-center group"> Data Enrichment <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="data-enrichment" aria-haspopup="dialog" aria-label="Share link: Data Enrichment"> Share link </button> </h4>Enrich data during import: <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Calculate derived fields LOAD CSV FROM '/data/users.csv' WITH HEADERS AS row CREATE (p:Person { id: row.id, first_name: row.first_name, last_name: row.last_name, full_name: row.first_name || ' ' || row.last_name, birth_date: DATE(row.birth_date), age: YEAR(NOW()) - YEAR(DATE(row.birth_date)) }); -- Lookup and link existing nodes LOAD CSV FROM '/data/purchases.csv' WITH HEADERS AS row MATCH (user:User {id: row.user_id}) MATCH (product:Product {id: row.product_id}) CREATE (user)-[:PURCHASED { date: DATE(row.purchase_date), amount: toDecimal(row.amount), quantity: toInteger(row.quantity) }]->(product); </code></pre></div> <h3 id="etl-pipeline-integration" class="position-relative d-flex align-items-center group"> ETL Pipeline Integration <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="etl-pipeline-integration" aria-haspopup="dialog" aria-label="Share link: ETL Pipeline Integration"> Share link </button> </h3> <h4 id="apache-airflow" class="position-relative d-flex align-items-center group"> Apache Airflow <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="apache-airflow" aria-haspopup="dialog" aria-label="Share link: Apache Airflow"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python">from airflow import DAG from airflow.operators.python import PythonOperator from datetime import datetime from geode_client import Client import pandas as pd def extract_transform_load(): # Extract from source df = pd.read_csv('source_data.csv') # Transform df['full_name'] = df['first_name'] + ' ' + df['last_name'] df['age'] = datetime.now().year - pd.to_datetime(df['birth_date']).dt.year # Load to Geode client = Client("localhost:3141") for _, row in df.iterrows(): client.execute(""" MERGE (p:Person {id: $id}) SET p.name = $name, p.age = $age, p.email = $email """, dict(row)) return len(df) with DAG('import_to_geode', start_date=datetime(2026, 1, 1)) as dag: import_task = PythonOperator( task_id='etl_to_geode', python_callable=extract_transform_load ) </code></pre></div> <h3 id="performance-optimization" class="position-relative d-flex align-items-center group"> Performance Optimization <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="performance-optimization" aria-haspopup="dialog" aria-label="Share link: Performance Optimization"> Share link </button> </h3> <h4 id="index-creation-strategy" class="position-relative d-flex align-items-center group"> Index Creation Strategy <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="index-creation-strategy" aria-haspopup="dialog" aria-label="Share link: Index Creation Strategy"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Create indexes AFTER import for better performance -- 1. Import data without indexes LOAD CSV FROM '/data/users.csv' WITH HEADERS AS row CREATE (p:Person {id: row.id, name: row.name, email: row.email}); -- 2. Create indexes after import CREATE INDEX idx_person_id ON Person(id); CREATE INDEX idx_person_email ON Person(email); CREATE INDEX idx_person_name ON Person(name); </code></pre></div> <h4 id="transaction-sizing" class="position-relative d-flex align-items-center group"> Transaction Sizing <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="transaction-sizing" aria-haspopup="dialog" aria-label="Share link: Transaction Sizing"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-python" data-lang="python"># Optimal batch size: 1000-10000 records OPTIMAL_BATCH_SIZE = 5000 async def optimized_import(data): client = Client(host="localhost", port=3141) async with client.connection() as conn: for i in range(0, len(data), OPTIMAL_BATCH_SIZE): batch = data[i:i+OPTIMAL_BATCH_SIZE] async with client.connection() as tx: await tx.begin() for record in batch: await tx.execute(create_query, record) await tx.commit() </code></pre></div> <h3 id="best-practices" class="position-relative d-flex align-items-center group"> Best Practices <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="best-practices" aria-haspopup="dialog" aria-label="Share link: Best Practices"> Share link </button> </h3><ol> <li>Validate Data: Check data quality before import</li> <li>Use Transactions: Batch operations in transactions</li> <li>Create Indexes After: Import first, index later</li> <li>Handle Duplicates: Use MERGE for idempotent imports</li> <li>Monitor Progress: Log import progress and errors</li> <li>Test with Sample: Validate on small dataset first</li> <li>Plan for Rollback: Keep backups before large imports</li> </ol> <h3 id="troubleshooting" class="position-relative d-flex align-items-center group"> Troubleshooting <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="troubleshooting" aria-haspopup="dialog" aria-label="Share link: Troubleshooting"> Share link </button> </h3> <h4 id="memory-issues" class="position-relative d-flex align-items-center group"> Memory Issues <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="memory-issues" aria-haspopup="dialog" aria-label="Share link: Memory Issues"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"># Use streaming for large files geode import --format jsonl --stream --input huge_file.jsonl # Or split into smaller chunks split -l 100000 large_file.csv chunk_ for chunk in chunk_*; do geode import --input "$chunk" done </code></pre></div> <h4 id="performance-problems" class="position-relative d-flex align-items-center group"> Performance Problems <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="performance-problems" aria-haspopup="dialog" aria-label="Share link: Performance Problems"> Share link </button> </h4><div class="highlight"><pre tabindex="0" class="chroma"><code class="language-gql" data-lang="gql">-- Disable constraints during import ALTER TABLE Person DISABLE CONSTRAINTS; -- Import data LOAD CSV FROM 'data.csv' ... -- Re-enable constraints ALTER TABLE Person ENABLE CONSTRAINTS; </code></pre></div> <h3 id="related-topics" class="position-relative d-flex align-items-center group"> Related Topics <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="related-topics" aria-haspopup="dialog" aria-label="Share link: Related Topics"> Share link </button> </h3><ul> <li><a href="/tags/export/" >Export</a> - Data export strategies</li> <li><a href="/tags/migration/" >Migration</a> - Database migration</li> <li><a href="/tags/etl/" >ETL</a> - ETL pipelines</li> <li><a href="/tags/integration/" >Integration</a> - Integration patterns</li> </ul> <h3 id="further-reading" class="position-relative d-flex align-items-center group"> Further Reading <button type="button" class="h-share btn btn-link p-0 text-decoration-none link-secondary opacity-50 hover-opacity-100 transition-all ms-1" data-share-target="further-reading" aria-haspopup="dialog" aria-label="Share link: Further Reading"> Share link </button> </h3><ul> <li><a href="https://www.postgresql.org/docs/current/populate.html" aria-label="Bulk Loading Best Practices – opens in new window" target="_blank" rel="noopener noreferrer" >Bulk Loading Best Practices ↗ </a> - PostgreSQL guide</li> <li><a href="https://www.kimballgroup.com/" aria-label="ETL Design Patterns – opens in new window" target="_blank" rel="noopener noreferrer" >ETL Design Patterns ↗ </a> - Kimball Group resources</li> </ul>

Popular

Data Import - Loading Data into Geode

Related Articles

Data Import Guide