Bulk Data Operations

Bulk Export ($export)

The Bulk Data Access specification enables efficient export of large datasets. Unlike standard FHIR queries that return individual resources, bulk export generates files containing thousands or millions of resources in NDJSON format. Use bulk export for analytics, population health, data warehousing, and backup scenarios where processing large volumes efficiently is essential.

System-Level Export

Export all data from the server. System-level export retrieves all resources of specified types across all patients. This is useful for initial data migration, analytics pipelines, and complete system backups. The export runs asynchronously, returning a status URL for polling until completion.

import ca.uhn.fhir.rest.client.api.IGenericClient;
import org.hl7.fhir.r4.model.Parameters;
import org.hl7.fhir.r4.model.InstantType;

public class BulkExportExample {

    private static final FhirContext ctx = FhirContext.forR4();

    public static String initiateSystemExport(String serverBase) {
        IGenericClient client = ctx.newRestfulGenericClient(serverBase);

        // Build export parameters
        Parameters params = new Parameters();
        params.addParameter()
            .setName("_outputFormat")
            .setValue(new StringType("application/fhir+ndjson"));
        params.addParameter()
            .setName("_since")
            .setValue(new InstantType("2023-01-01T00:00:00Z"));
        params.addParameter()
            .setName("_type")
            .setValue(new StringType("Patient,Observation,Condition"));

        // Initiate export
        try {
            client.operation()
                .onServer()
                .named("$export")
                .withParameters(params)
                .withAdditionalHeader("Prefer", "respond-async")
                .execute();

            // Get content-location from response header
            // This will contain the status polling URL
            String statusUrl = getContentLocationHeader();
            System.out.println("Export initiated. Status URL: " + statusUrl);

            return statusUrl;

        } catch (Exception e) {
            System.err.println("Export failed: " + e.getMessage());
            return null;
        }
    }

    private static String getContentLocationHeader() {
        // Extract from HTTP response headers
        return "http://example.com/export-status/123"; // Placeholder
    }
}

Patient-Level Export

Export data for a specific patient. Patient-level export retrieves all resources in a patient’s compartment, similar to $everything but in bulk format. Use this when you need a complete patient record for data portability, care transitions, or patient-facing applications that process data offline.

public static String initiatePatientExport(String serverBase, String patientId) {
    IGenericClient client = ctx.newRestfulGenericClient(serverBase);

    Parameters params = new Parameters();
    params.addParameter()
        .setName("_outputFormat")
        .setValue(new StringType("application/fhir+ndjson"));

    // Patient-level export
    client.operation()
        .onInstance(new IdType("Patient", patientId))
        .named("$export")
        .withParameters(params)
        .withAdditionalHeader("Prefer", "respond-async")
        .execute();

    return getContentLocationHeader();
}

Group-Level Export

Export data for all patients in a group. Group export is powerful for cohort analysis, research studies, and population health management. Define groups based on conditions, care teams, or custom criteria, then export all related data efficiently. This approach scales better than querying individual patients.

public static String initiateGroupExport(String serverBase, String groupId) {
    IGenericClient client = ctx.newRestfulGenericClient(serverBase);

    Parameters params = new Parameters();
    params.addParameter()
        .setName("_outputFormat")
        .setValue(new StringType("application/fhir+ndjson"));
    params.addParameter()
        .setName("_type")
        .setValue(new StringType("Patient,Observation,MedicationRequest"));

    // Group-level export
    client.operation()
        .onInstance(new IdType("Group", groupId))
        .named("$export")
        .withParameters(params)
        .withAdditionalHeader("Prefer", "respond-async")
        .execute();

    return getContentLocationHeader();
}

Polling Export Status

Monitor export progress and retrieve results. Bulk exports can take minutes to hours depending on data volume. Poll the status URL periodically, respecting the Retry-After header to avoid overwhelming the server. When complete, the response includes a manifest with URLs to download the generated files.

import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.util.EntityUtils;

public static ExportStatus checkExportStatus(String statusUrl) {
    try {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet request = new HttpGet(statusUrl);
        CloseableHttpResponse response = httpClient.execute(request);

        int statusCode = response.getStatusLine().getStatusCode();

        if (statusCode == 202) {
            // Still in progress
            String retryAfter = response.getFirstHeader("Retry-After").getValue();
            return new ExportStatus(false, null, Integer.parseInt(retryAfter));

        } else if (statusCode == 200) {
            // Complete - parse response
            String responseBody = EntityUtils.toString(response.getEntity());
            ExportManifest manifest = parseExportManifest(responseBody);
            return new ExportStatus(true, manifest, 0);
        }

    } catch (Exception e) {
        System.err.println("Error checking status: " + e.getMessage());
    }

    return null;
}

public static void downloadExportFiles(ExportManifest manifest, String outputDir) {
    for (ExportFile file : manifest.getOutput()) {
        String url = file.getUrl();
        String type = file.getType();

        System.out.println("Downloading " + type + " from " + url);

        try {
            downloadFile(url, outputDir + "/" + type + ".ndjson");
        } catch (Exception e) {
            System.err.println("Download failed: " + e.getMessage());
        }
    }
}

private static void downloadFile(String url, String outputPath) throws Exception {
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpGet request = new HttpGet(url);

    CloseableHttpResponse response = httpClient.execute(request);

    try (FileOutputStream fos = new FileOutputStream(outputPath)) {
        response.getEntity().writeTo(fos);
    }

    System.out.println("Downloaded to: " + outputPath);
}

private static ExportManifest parseExportManifest(String json) {
    // Parse JSON manifest containing file URLs
    ObjectMapper mapper = new ObjectMapper();
    return mapper.readValue(json, ExportManifest.class);
}

static class ExportStatus {
    private boolean complete;
    private ExportManifest manifest;
    private int retryAfter;

    public ExportStatus(boolean complete, ExportManifest manifest, int retryAfter) {
        this.complete = complete;
        this.manifest = manifest;
        this.retryAfter = retryAfter;
    }

    public boolean isComplete() { return complete; }
    public ExportManifest getManifest() { return manifest; }
    public int getRetryAfter() { return retryAfter; }
}

static class ExportManifest {
    private List<ExportFile> output;

    public List<ExportFile> getOutput() { return output; }
}

static class ExportFile {
    private String type;
    private String url;

    public String getType() { return type; }
    public String getUrl() { return url; }
}

Processing NDJSON Files

NDJSON (Newline Delimited JSON) contains one resource per line. This format enables streaming processing of large files without loading everything into memory. Each line is a complete, valid JSON object representing a FHIR resource. Process files line by line for memory efficiency, parsing each resource independently.

import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.*;
import java.nio.file.*;

public class NDJsonProcessingExample {

    private static final FhirContext ctx = FhirContext.forR4();
    private static final IParser jsonParser = ctx.newJsonParser();

    public static void processNDJsonFile(String filePath) throws IOException {

        Path path = Paths.get(filePath);

        try (BufferedReader reader = Files.newBufferedReader(path)) {
            String line;
            int count = 0;

            while ((line = reader.readLine()) != null) {
                if (line.trim().isEmpty()) {
                    continue;
                }

                // Parse each line as a FHIR resource
                try {
                    IBaseResource resource = jsonParser.parseResource(line);

                    // Process the resource
                    processResource(resource);
                    count++;

                } catch (Exception e) {
                    System.err.println("Error parsing line: " + e.getMessage());
                }
            }

            System.out.println("Processed " + count + " resources");
        }
    }

    private static void processResource(IBaseResource resource) {
        if (resource instanceof Patient) {
            processPatient((Patient) resource);
        } else if (resource instanceof Observation) {
            processObservation((Observation) resource);
        }
        // Add more resource types as needed
    }

    private static void processPatient(Patient patient) {
        System.out.println("Patient: " + patient.getId() +
            " - " + patient.getNameFirstRep().getNameAsSingleString());
    }

    private static void processObservation(Observation obs) {
        System.out.println("Observation: " + obs.getId() +
            " - " + obs.getCode().getText());
    }

    public static void writeToNDJson(List<IBaseResource> resources, String outputPath)
            throws IOException {

        try (BufferedWriter writer = Files.newBufferedWriter(Paths.get(outputPath))) {
            for (IBaseResource resource : resources) {
                String json = jsonParser.encodeResourceToString(resource);
                writer.write(json);
                writer.newLine();
            }
        }

        System.out.println("Wrote " + resources.size() + " resources to " + outputPath);
    }

    public static void main(String[] args) throws IOException {
        processNDJsonFile("export/Patient.ndjson");
    }
}

Bulk Import

Import large datasets efficiently. While the FHIR specification focuses on export, practical implementations need import capabilities. Use transaction bundles to upload resources in batches, balancing throughput against server constraints. Implement error handling to identify and retry failed batches without reprocessing successful ones.

public class BulkImportExample {

    private static final FhirContext ctx = FhirContext.forR4();

    public static void bulkImportFromNDJson(String serverBase, String ndjsonFile)
            throws IOException {

        IGenericClient client = ctx.newRestfulGenericClient(serverBase);
        IParser parser = ctx.newJsonParser();

        List<IBaseResource> batch = new ArrayList<>();
        int batchSize = 100;
        int totalProcessed = 0;

        try (BufferedReader reader = Files.newBufferedReader(Paths.get(ndjsonFile))) {
            String line;

            while ((line = reader.readLine()) != null) {
                if (line.trim().isEmpty()) continue;

                IBaseResource resource = parser.parseResource(line);
                batch.add(resource);

                if (batch.size() >= batchSize) {
                    // Create transaction bundle
                    Bundle bundle = createTransactionBundle(batch);

                    // Execute
                    try {
                        client.transaction().withBundle(bundle).execute();
                        totalProcessed += batch.size();
                        System.out.println("Imported " + totalProcessed + " resources");
                    } catch (Exception e) {
                        System.err.println("Batch import failed: " + e.getMessage());
                    }

                    batch.clear();
                }
            }

            // Import remaining resources
            if (!batch.isEmpty()) {
                Bundle bundle = createTransactionBundle(batch);
                client.transaction().withBundle(bundle).execute();
                totalProcessed += batch.size();
            }
        }

        System.out.println("Total imported: " + totalProcessed + " resources");
    }

    private static Bundle createTransactionBundle(List<IBaseResource> resources) {
        Bundle bundle = new Bundle();
        bundle.setType(Bundle.BundleType.TRANSACTION);

        for (IBaseResource resource : resources) {
            bundle.addEntry()
                .setResource((Resource) resource)
                .getRequest()
                    .setMethod(Bundle.HTTPVerb.POST)
                    .setUrl(resource.fhirType());
        }

        return bundle;
    }
}

Streaming Large Files

Process large NDJSON files efficiently with streaming. For files with millions of resources, sequential processing becomes slow. Use parallel streaming with thread pools to process multiple resources concurrently. The Java Stream API combined with CompletableFuture enables efficient parallel processing while maintaining manageable memory usage.

import java.util.concurrent.*;
import java.util.stream.*;

public class StreamingNDJsonProcessor {

    private static final FhirContext ctx = FhirContext.forR4();
    private static final int THREAD_POOL_SIZE = 4;

    public static void processLargeFile(String filePath) throws Exception {
        ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
        IParser parser = ctx.newJsonParser();

        AtomicInteger processedCount = new AtomicInteger(0);

        try (Stream<String> lines = Files.lines(Paths.get(filePath))) {
            List<CompletableFuture<Void>> futures = lines
                .filter(line -> !line.trim().isEmpty())
                .map(line -> CompletableFuture.runAsync(() -> {
                    try {
                        IBaseResource resource = parser.parseResource(line);
                        processResourceAsync(resource);
                        int count = processedCount.incrementAndGet();
                        if (count % 1000 == 0) {
                            System.out.println("Processed " + count + " resources");
                        }
                    } catch (Exception e) {
                        System.err.println("Error: " + e.getMessage());
                    }
                }, executor))
                .collect(Collectors.toList());

            // Wait for all to complete
            CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
        }

        executor.shutdown();
        executor.awaitTermination(1, TimeUnit.HOURS);

        System.out.println("Total processed: " + processedCount.get());
    }

    private static void processResourceAsync(IBaseResource resource) {
        // Process resource (e.g., store in database, transform, etc.)
    }
}

Export Workflow Example

Complete export workflow with error handling. Production implementations need robust workflows that handle timeouts, network failures, and partial completions. Structure your workflow with clear stages: initiation, polling, download, and processing. Implement checkpointing to resume interrupted workflows without starting over.

public class CompleteExportWorkflow {

    private static final FhirContext ctx = FhirContext.forR4();

    public static void runExport(String serverBase, String outputDir) {
        try {
            // 1. Initiate export
            System.out.println("Initiating export...");
            String statusUrl = initiateSystemExport(serverBase);

            if (statusUrl == null) {
                throw new RuntimeException("Failed to initiate export");
            }

            // 2. Poll for completion
            System.out.println("Waiting for export to complete...");
            ExportStatus status;
            do {
                status = checkExportStatus(statusUrl);

                if (!status.isComplete()) {
                    int waitSeconds = Math.max(status.getRetryAfter(), 10);
                    System.out.println("Export in progress. Waiting " +
                        waitSeconds + " seconds...");
                    Thread.sleep(waitSeconds * 1000L);
                }
            } while (!status.isComplete());

            // 3. Download files
            System.out.println("Export complete. Downloading files...");
            ExportManifest manifest = status.getManifest();

            Files.createDirectories(Paths.get(outputDir));

            for (ExportFile file : manifest.getOutput()) {
                System.out.println("Downloading: " + file.getType());
                downloadFile(file.getUrl(),
                    outputDir + "/" + file.getType() + ".ndjson");
            }

            // 4. Process downloaded files
            System.out.println("Processing downloaded files...");
            for (ExportFile file : manifest.getOutput()) {
                String filePath = outputDir + "/" + file.getType() + ".ndjson";
                processNDJsonFile(filePath);
            }

            System.out.println("Export workflow complete!");

        } catch (Exception e) {
            System.err.println("Export workflow failed: " + e.getMessage());
            e.printStackTrace();
        }
    }
}

Bulk Data Parameters