UI-TARS Desktop: ByteDance's Revolutionary Multimodal AI Agent for Complete Computer Control

DESKTOP_ACTIONS = [
    'click(x, y)',           # Single click at coordinates
    'double_click(x, y)',    # Double click
    'right_click(x, y)',     # Right click
    'drag(x1, y1, x2, y2)',  # Drag from point to point
    'type(text)',            # Type text
    'key(combination)',      # Keyboard shortcuts
    'scroll(direction)',     # Scroll up/down
    'wait(seconds)',         # Wait for loading
    'screenshot()',          # Capture screen
    'finished()'            # Task completion
]

MOBILE_ACTIONS = [
    'tap(x, y)',
    'long_press(x, y)',
    'swipe(x1, y1, x2, y2)',
    'open_app(name)',
    'press_home()',
    'press_back()',
    'type(text)',
    'finished()'
]

# Download from GitHub releases
wget https://github.com/bytedance/UI-TARS-desktop/releases/latest/download/UI-TARS-Desktop-[version].[platform]

# macOS with Homebrew
brew install --cask ui-tars-desktop

# Windows with Chocolatey
choco install ui-tars-desktop

# AppImage installation
chmod +x UI-TARS-Desktop-[version].AppImage
./UI-TARS-Desktop-[version].AppImage

# Snap installation
sudo snap install ui-tars-desktop

# Quick start with npx (no installation)
npx @agent-tars/cli@latest

# Global installation (requires Node.js >= 22)
npm install @agent-tars/cli@latest -g

# Run with model provider
agent-tars --provider anthropic \
           --model claude-3-7-sonnet-latest \
           --apiKey your-api-key

# Or with Volcengine
agent-tars --provider volcengine \
           --model doubao-1-5-thinking-vision-pro-250428 \
           --apiKey your-api-key

# Install the SDK
npm install @ui-tars/sdk @ui-tars/operator-nut-js

import { GUIAgent } from '@ui-tars/sdk';
import { NutJSOperator } from '@ui-tars/operator-nut-js';

// Initialize the agent
const guiAgent = new GUIAgent({
  model: {
    baseURL: 'https://your-model-endpoint/v1',
    apiKey: 'your-api-key',
    model: 'ui-tars-1.5-7b',
  },
  operator: new NutJSOperator(),
  onData: ({ data }) => {
    console.log('Action:', data);
  },
  onError: ({ error }) => {
    console.error('Error:', error);
  },
});

// Execute a task
await guiAgent.run('Open VS Code and create a new Python file');

class TaskAutomation {
  constructor(config) {
    this.agent = new GUIAgent({
      model: config.model,
      operator: new NutJSOperator(),
      signal: this.abortController.signal,
    });
  }

  async bookFlight(details) {
    const instruction = `
      1. Open Chrome browser
      2. Navigate to ${details.website}
      3. Search for flights from ${details.from} to ${details.to}
      4. Select departure date: ${details.departDate}
      5. Select return date: ${details.returnDate}
      6. Choose the cheapest available option
      7. Fill passenger details: ${details.passenger}
      8. Complete booking (stop before payment)
    `;
    
    return await this.agent.run(instruction);
  }

  async generateReport(data) {
    const steps = [
      'Open Microsoft Excel',
      `Create pivot table from data: ${JSON.stringify(data)}`,
      'Generate charts for key metrics',
      'Export as PDF report',
      'Email to team@company.com'
    ];
    
    for (const step of steps) {
      await this.agent.run(step);
      await this.waitForCompletion();
    }
  }

  abort() {
    this.abortController.abort();
  }
}

import { Operator } from '@ui-tars/sdk/core';

export class CustomOperator extends Operator {
  // Define action spaces for the model
  static MANUAL = {
    ACTION_SPACES: [
      'click(coordinates="x,y")',
      'type(content="text")',
      'scroll(direction="up|down")',
      'custom_action(params="...")',
      'finished()'
    ],
  };

  async screenshot() {
    // Capture current screen state
    const screenCapture = await this.captureScreen();
    return {
      base64: screenCapture.toBase64(),
      scaleFactor: this.getDevicePixelRatio()
    };
  }

  async execute(params) {
    const { action, args } = this.parseAction(params.content);
    
    switch(action) {
      case 'click':
        return await this.performClick(args);
      case 'type':
        return await this.performTyping(args);
      case 'custom_action':
        return await this.performCustomAction(args);
      default:
        throw new Error(`Unknown action: ${action}`);
    }
  }

  async performCustomAction(args) {
    // Implement your custom logic
    console.log('Executing custom action:', args);
    return { success: true, data: args };
  }
}

// Configuration for different providers
const providers = {
  huggingface: {
    provider: 'Hugging Face for UI-TARS-1.5',
    baseURL: 'https://your-endpoint.endpoints.huggingface.cloud/v1/',
    apiKey: 'hf_xxxxxxxxxxxxx',
    model: 'ui-tars-1.5-7b'
  },
  volcengine: {
    provider: 'VolcEngine Ark for Doubao-1.5-UI-TARS',
    baseURL: 'https://ark.cn-beijing.volces.com/api/v3',
    apiKey: 'ARK_API_KEY',
    model: 'doubao-1.5-ui-tars-250328'
  },
  local: {
    provider: 'Local Deployment',
    baseURL: 'http://localhost:8080/v1',
    apiKey: 'not-required',
    model: 'ui-tars-local'
  }
};

{
  "vlm": {
    "provider": "Hugging Face for UI-TARS-1.5",
    "baseURL": "https://api.example.com/v1",
    "apiKey": "your-api-key",
    "model": "ui-tars-1.5-7b",
    "language": "en"
  },
  "execution": {
    "maxSteps": 10,
    "loopWaitTime": 3000,
    "screenshotDelay": 500,
    "actionTimeout": 30000
  },
  "features": {
    "useResponseAPI": false,
    "enableTelemetry": false,
    "debugMode": false
  }
}

class UITestAutomation {
  async testLoginFlow() {
    const testCases = [
      {
        description: 'Valid login',
        username: 'test@example.com',
        password: 'validPassword',
        expectedResult: 'Dashboard visible'
      },
      {
        description: 'Invalid credentials',
        username: 'test@example.com',
        password: 'wrongPassword',
        expectedResult: 'Error message displayed'
      }
    ];

    for (const testCase of testCases) {
      await this.agent.run(`
        1. Open application
        2. Click login button
        3. Enter username: ${testCase.username}
        4. Enter password: ${testCase.password}
        5. Click submit
        6. Verify: ${testCase.expectedResult}
      `);
      
      await this.captureTestResult(testCase);
    }
  }
}

class DataEntryAutomation {
  async processInvoices(invoices) {
    for (const invoice of invoices) {
      const instruction = `
        Open accounting software
        Navigate to 'New Invoice'
        Fill in:
        - Customer: ${invoice.customer}
        - Amount: ${invoice.amount}
        - Date: ${invoice.date}
        - Items: ${invoice.items.join(', ')}
        Save and generate PDF
        Email to ${invoice.email}
      `;
      
      await this.agent.run(instruction);
    }
  }
}

class CreativeWorkflow {
  async exportDesignAssets() {
    await this.agent.run(`
      1. Open Photoshop project 'brand-assets.psd'
      2. Export all layers as PNG files
      3. Open After Effects
      4. Import the exported PNGs
      5. Create composition with imported assets
      6. Apply preset animation 'slide-in'
      7. Export as MP4 with H.264 codec
      8. Upload to project folder on Google Drive
    `);
  }
}

// Remote Computer Control
const remoteComputer = new RemoteOperator({
  type: 'computer',
  region: 'us-west-2',
  sessionDuration: 30 // minutes
});

await remoteComputer.connect();
await remoteComputer.execute('Install and configure Docker');

// Remote Browser Control
const remoteBrowser = new RemoteOperator({
  type: 'browser',
  browserType: 'chrome',
  headless: false
});

await remoteBrowser.navigate('https://example.com');
await remoteBrowser.execute('Complete the registration form');

// Mount MCP servers for extended functionality
const agent = new GUIAgent({
  model: config.model,
  operator: new NutJSOperator(),
  mcpServers: [
    {
      name: 'weather',
      endpoint: 'https://mcp-weather.example.com',
      capabilities: ['get_weather', 'forecast']
    },
    {
      name: 'database',
      endpoint: 'https://mcp-db.example.com',
      capabilities: ['query', 'update']
    }
  ]
});

// Use MCP tools in instructions
await agent.run(`
  Get current weather for San Francisco
  If temperature > 70°F, book outdoor restaurant
  Otherwise, find indoor venue
  Update database with booking details
`);

# Clone and build from source
git clone https://github.com/bytedance/UI-TARS-desktop.git
cd UI-TARS-desktop
npm install
npm run build

# Run locally
npm start

FROM node:22-alpine

WORKDIR /app

COPY package*.json ./
RUN npm ci --only=production

COPY . .

EXPOSE 8080

CMD ["npm", "run", "serve"]

# app.py for Hugging Face deployment
import gradio as gr
from ui_tars import UITARSModel

model = UITARSModel.from_pretrained("ByteDance-Seed/UI-TARS-1.5-7B")

def process_instruction(instruction, screenshot):
    result = model.execute(
        instruction=instruction,
        screenshot=screenshot
    )
    return result['action'], result['visualization']

iface = gr.Interface(
    fn=process_instruction,
    inputs=[
        gr.Textbox(label="Instruction"),
        gr.Image(label="Screenshot")
    ],
    outputs=[
        gr.Textbox(label="Action"),
        gr.Image(label="Result")
    ]
)

iface.launch()

// Good: Clear, step-by-step instructions
const goodInstruction = `
  1. Open Microsoft Word
  2. Create new document
  3. Type heading: "Monthly Report"
  4. Apply Heading 1 style
  5. Save as "report_2025.docx"
`;

// Bad: Vague, ambiguous instructions
const badInstruction = "Make a report in Word";

class RobustAgent {
  async executeWithRetry(instruction, maxRetries = 3) {
    for (let attempt = 1; attempt <= maxRetries; attempt++) {
      try {
        return await this.agent.run(instruction);
      } catch (error) {
        console.log(`Attempt ${attempt} failed:`, error);
        
        if (attempt === maxRetries) {
          throw new Error(`Failed after ${maxRetries} attempts`);
        }
        
        // Wait before retry with exponential backoff
        await this.wait(Math.pow(2, attempt) * 1000);
      }
    }
  }
}

// Batch operations for efficiency
const batchOperations = [
  'Open all Excel files in folder',
  'Apply formatting template to each',
  'Export all as PDFs',
  'Close Excel'
];

// Execute as single instruction
await agent.run(batchOperations.join('\n'));

const secureConfig = {
  sandbox: true,
  permissions: {
    fileSystem: ['read', 'write'],
    network: ['localhost'],
    clipboard: false,
    camera: false,
    microphone: false
  },
  blockedApplications: [
    'System Preferences',
    'Terminal',
    'Command Prompt'
  ]
};

class AuditedAgent extends GUIAgent {
  async run(instruction) {
    const startTime = Date.now();
    const result = await super.run(instruction);
    
    await this.logAudit({
      timestamp: new Date().toISOString(),
      instruction,
      duration: Date.now() - startTime,
      actions: result.actions,
      user: process.env.USER
    });
    
    return result;
  }
}

# Fork and contribute
git clone https://github.com/YOUR_USERNAME/UI-TARS-desktop.git
cd UI-TARS-desktop
git checkout -b feature/your-feature

# Make changes and test
npm test
npm run lint

# Submit pull request
git push origin feature/your-feature