API Integration

UnDatasIO provides comprehensive APIs and SDKs to integrate file processing and data transformation capabilities into your applications. Our RESTful API and client libraries support multiple programming languages and platforms.

RESTful API

Base URL

https://api.undatasio.com/v1

Authentication

All API requests require authentication using API keys:

# Include in headers
Authorization: Bearer YOUR_API_KEY

# Or as query parameter
?api_key=YOUR_API_KEY

Rate Limits

• Free Tier: 1,000 requests per hour
• Pro Tier: 10,000 requests per hour
• Enterprise: Custom limits

Core Endpoints

File Upload

Upload files for processing:

POST /files/upload
Content-Type: multipart/form-data

# Form data
file: [binary file data]
options: {
  "autoDetect": true,
  "validateSchema": true
}

Response:

{
  "fileId": "f_1234567890",
  "filename": "data.csv",
  "size": 1024,
  "status": "uploaded",
  "uploadedAt": "2024-01-15T10:30:00Z"
}

File Parsing

Parse uploaded files:

POST /files/{fileId}/parse
Content-Type: application/json

{
  "format": "csv",
  "options": {
    "delimiter": "comma",
    "hasHeader": true,
    "encoding": "UTF-8"
  }
}

Response:

{
  "parseId": "p_1234567890",
  "status": "completed",
  "result": {
    "rows": 1000,
    "columns": 5,
    "preview": [...],
    "schema": {...}
  },
  "completedAt": "2024-01-15T10:31:00Z"
}

Data Processing

Process parsed data:

POST /data/{parseId}/process
Content-Type: application/json

{
  "operations": [
    {
      "type": "filter",
      "condition": {
        "column": "age",
        "operator": ">=",
        "value": 18
      }
    },
    {
      "type": "transform",
      "column": "name",
      "operation": "uppercase"
    }
  ]
}

Data Export

Export processed data:

GET /data/{processId}/export
?format=csv&includeHeader=true

SDK Libraries

JavaScript/Node.js

Install the SDK:

npm install @undatasio/sdk

Basic Usage:

import { UnDatasIO } from "@undatasio/sdk";

const client = new UnDatasIO({
  apiKey: "your_api_key_here",
  baseUrl: "https://api.undatasio.com/v1",
});

// Upload and parse a file
const file = await client.uploadFile("data.csv", {
  autoDetect: true,
  validateSchema: true,
});

const parsed = await client.parseFile(file.fileId, {
  format: "csv",
  delimiter: "comma",
  hasHeader: true,
});

// Process the data
const processed = await client.processData(parsed.parseId, {
  operations: [
    {
      type: "filter",
      condition: { column: "age", operator: ">=", value: 18 },
    },
  ],
});

// Export results
const csvData = await client.exportData(processed.processId, {
  format: "csv",
  includeHeader: true,
});

Advanced Features:

// Batch processing
const results = await client.batchProcess(
  ["file1.csv", "file2.json", "file3.xlsx"],
  {
    operations: [
      { type: "clean", strategy: "remove_duplicates" },
      { type: "validate", schema: userSchema },
    ],
  }
);

// Streaming upload
const stream = client.createUploadStream("large-file.csv");
stream.on("progress", (progress) => {
  console.log(`Upload progress: ${progress}%`);
});

// Real-time processing
const processor = client.createRealTimeProcessor({
  onData: (data) => {
    console.log("Processed data:", data);
  },
  onError: (error) => {
    console.error("Processing error:", error);
  },
});

Python

Install the SDK:

pip install undatasio-sdk

Basic Usage:

from undatasio import UnDatasIO

client = UnDatasIO(api_key='your_api_key_here')

# Upload and parse a file
with open('data.csv', 'rb') as file:
    result = client.upload_file(
        file=file,
        options={
            'auto_detect': True,
            'validate_schema': True
        }
    )

parsed = client.parse_file(
    file_id=result['fileId'],
    options={
        'format': 'csv',
        'delimiter': 'comma',
        'has_header': True
    }
)

# Process the data
processed = client.process_data(
    parse_id=parsed['parseId'],
    operations=[
        {
            'type': 'filter',
            'condition': {
                'column': 'age',
                'operator': '>=',
                'value': 18
            }
        }
    ]
)

# Export results
csv_data = client.export_data(
    process_id=processed['processId'],
    format='csv',
    include_header=True
)

Advanced Features:

# Async processing
import asyncio
from undatasio import AsyncUnDatasIO

async def process_files():
    client = AsyncUnDatasIO(api_key='your_api_key_here')

    tasks = []
    for filename in ['file1.csv', 'file2.json']:
        task = client.upload_and_process(filename)
        tasks.append(task)

    results = await asyncio.gather(*tasks)
    return results

# Data pipeline
pipeline = client.create_pipeline([
    {'type': 'upload', 'file': 'data.csv'},
    {'type': 'parse', 'format': 'csv'},
    {'type': 'clean', 'strategy': 'remove_duplicates'},
    {'type': 'transform', 'operations': [...]},
    {'type': 'export', 'format': 'json'}
])

result = await pipeline.execute()

Java

Add to your pom.xml:

<dependency>
    <groupId>com.undatasio</groupId>
    <artifactId>undatasio-sdk</artifactId>
    <version>1.0.0</version>
</dependency>

Basic Usage:

import com.undatasio.UnDatasIO;
import com.undatasio.models.*;

UnDatasIO client = new UnDatasIO("your_api_key_here");

// Upload and parse a file
FileUploadRequest uploadRequest = FileUploadRequest.builder()
    .file(new File("data.csv"))
    .autoDetect(true)
    .validateSchema(true)
    .build();

FileUploadResponse uploadResponse = client.uploadFile(uploadRequest);

// Parse the file
ParseRequest parseRequest = ParseRequest.builder()
    .fileId(uploadResponse.getFileId())
    .format("csv")
    .delimiter("comma")
    .hasHeader(true)
    .build();

ParseResponse parseResponse = client.parseFile(parseRequest);

// Process the data
List<Operation> operations = Arrays.asList(
    Operation.builder()
        .type("filter")
        .condition(Condition.builder()
            .column("age")
            .operator(">=")
            .value(18)
            .build())
        .build()
);

ProcessRequest processRequest = ProcessRequest.builder()
    .parseId(parseResponse.getParseId())
    .operations(operations)
    .build();

ProcessResponse processResponse = client.processData(processRequest);

// Export results
ExportRequest exportRequest = ExportRequest.builder()
    .processId(processResponse.getProcessId())
    .format("csv")
    .includeHeader(true)
    .build();

byte[] csvData = client.exportData(exportRequest);

Go

Install the SDK:

go get github.com/undatasio/go-sdk

Basic Usage:

package main

import (
    "fmt"
    "log"
    "github.com/undatasio/go-sdk"
)

func main() {
    client := undatasio.NewClient("your_api_key_here")

    // Upload and parse a file
    file, err := os.Open("data.csv")
    if err != nil {
        log.Fatal(err)
    }
    defer file.Close()

    uploadResp, err := client.UploadFile(&undatasio.UploadRequest{
        File:        file,
        AutoDetect:  true,
        ValidateSchema: true,
    })
    if err != nil {
        log.Fatal(err)
    }

    // Parse the file
    parseResp, err := client.ParseFile(&undatasio.ParseRequest{
        FileID:     uploadResp.FileID,
        Format:     "csv",
        Delimiter:  "comma",
        HasHeader:  true,
    })
    if err != nil {
        log.Fatal(err)
    }

    // Process the data
    processResp, err := client.ProcessData(&undatasio.ProcessRequest{
        ParseID: parseResp.ParseID,
        Operations: []undatasio.Operation{
            {
                Type: "filter",
                Condition: undatasio.Condition{
                    Column:   "age",
                    Operator: ">=",
                    Value:    18,
                },
            },
        },
    })
    if err != nil {
        log.Fatal(err)
    }

    // Export results
    csvData, err := client.ExportData(&undatasio.ExportRequest{
        ProcessID:    processResp.ProcessID,
        Format:       "csv",
        IncludeHeader: true,
    })
    if err != nil {
        log.Fatal(err)
    }

    fmt.Printf("Exported data: %s\n", string(csvData))
}

Webhooks

Set up webhooks for real-time notifications:

POST /webhooks
Content-Type: application/json

{
  "url": "https://your-app.com/webhook",
  "events": ["file.uploaded", "file.parsed", "data.processed"],
  "secret": "your_webhook_secret"
}

Webhook Payload:

{
  "event": "file.parsed",
  "timestamp": "2024-01-15T10:31:00Z",
  "data": {
    "fileId": "f_1234567890",
    "parseId": "p_1234567890",
    "status": "completed",
    "result": {
      "rows": 1000,
      "columns": 5
    }
  },
  "signature": "sha256=..."
}

Error Handling

HTTP Status Codes

• 200: Success
• 400: Bad Request (invalid parameters)
• 401: Unauthorized (invalid API key)
• 403: Forbidden (insufficient permissions)
• 404: Not Found (resource doesn’t exist)
• 429: Too Many Requests (rate limit exceeded)
• 500: Internal Server Error

Error Response Format

{
  "error": {
    "code": "INVALID_FILE_FORMAT",
    "message": "Unsupported file format",
    "details": {
      "fileType": "unknown",
      "supportedFormats": ["csv", "json", "xlsx"]
    }
  }
}

SDK Error Handling

try {
  const result = await client.uploadFile("invalid.txt");
} catch (error) {
  if (error.code === "INVALID_FILE_FORMAT") {
    console.log("File format not supported");
  } else if (error.code === "RATE_LIMIT_EXCEEDED") {
    console.log("Rate limit exceeded, retry later");
  } else {
    console.log("Unexpected error:", error.message);
  }
}

Best Practices

API Usage

1. Use HTTPS: Always use HTTPS for API requests
2. Handle Rate Limits: Implement exponential backoff for retries
3. Validate Inputs: Validate file formats and parameters
4. Monitor Usage: Track API usage and costs
5. Error Handling: Implement comprehensive error handling

Performance Optimization

1. Batch Operations: Use batch endpoints for multiple files
2. Streaming: Use streaming for large files
3. Caching: Cache results when appropriate
4. Async Processing: Use async operations for better performance

Security

1. Secure API Keys: Store API keys securely
2. Webhook Validation: Validate webhook signatures
3. Input Sanitization: Sanitize all inputs
4. HTTPS Only: Use HTTPS for all communications

SDK Examples

File Processing Pipeline

// Complete file processing pipeline
async function processFile(filename, operations) {
  const client = new UnDatasIO({ apiKey: "your_api_key" });

  try {
    // Upload file
    const upload = await client.uploadFile(filename);
    console.log("File uploaded:", upload.fileId);

    // Parse file
    const parsed = await client.parseFile(upload.fileId);
    console.log("File parsed:", parsed.parseId);

    // Process data
    const processed = await client.processData(parsed.parseId, operations);
    console.log("Data processed:", processed.processId);

    // Export results
    const exported = await client.exportData(processed.processId, {
      format: "csv",
      includeHeader: true,
    });

    return exported;
  } catch (error) {
    console.error("Processing failed:", error);
    throw error;
  }
}

// Usage
const operations = [
  { type: "filter", condition: { column: "age", operator: ">=", value: 18 } },
  { type: "transform", column: "name", operation: "uppercase" },
];

processFile("users.csv", operations)
  .then((result) => console.log("Processing complete"))
  .catch((error) => console.error("Error:", error));

Real-time Processing

// Real-time file processing with webhooks
const client = new UnDatasIO({ apiKey: "your_api_key" });

// Set up webhook handler
app.post("/webhook", (req, res) => {
  const { event, data } = req.body;

  switch (event) {
    case "file.parsed":
      // Start processing when file is parsed
      client
        .processData(data.parseId, operations)
        .then((result) => console.log("Processing started:", result.processId));
      break;

    case "data.processed":
      // Export when processing is complete
      client
        .exportData(data.processId, { format: "json" })
        .then((result) => console.log("Export complete"));
      break;
  }

  res.status(200).send("OK");
});

Troubleshooting

Common Issues

Authentication Errors:

• Verify API key is correct
• Check API key permissions
• Ensure API key is not expired

File Upload Issues:

• Check file size limits
• Verify supported file formats
• Ensure proper file encoding

Processing Errors:

• Validate operation parameters
• Check data schema compatibility
• Review error messages for details

Getting Help

• API Documentation
• SDK Documentation
• Community Forum
• Support Team