Puppeteer Plus Martech MCP

🚀 Puppeteer

Puppeteer is a Node.js library developed by Google for controlling browsers, supporting Chromium and Chrome. It enables developers to automate tasks such as web scraping, screenshot and video generation, and performance analysis.

🚀 Quick Start

Create a Basic Project

Create a new Node.js project:

mkdir my-puppeteer-project && cd my-puppeteer-project
npm init -y

Install Puppeteer:

npm install puppeteer

Write Your First Script

Add the following code to the index.js file:

const puppeteer = require('puppeteer');

async function main() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  await page.screenshot({ path: 'example.png', format: 'png' });
  await browser.close();
}

main().catch(console.error);

Run the script:

node index.js

Advanced Features

Puppeteer provides powerful APIs to handle various browser tasks, for example:

Navigation and Page Operations

const puppeteer = require('puppeteer');

async function navigateExample() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  console.log(await page.title());
  await page.goBack();
  await page.goForward();
  await browser.close();
}
navigateExample().catch(console.error);

DOM Manipulation

const puppeteer = require('puppeteer');

async function domManipulation() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  
  // Get an element
  const element = await page.$('#element-id');
  console.log(await element.getPropertyValue('textContent'));
  
  // Simulate a click
  await element.click();
  
  // Input text
  const input = await page.$('input[type="text"]');
  await input.type('测试输入');
  
  await browser.close();
}
domManipulation().catch(console.error);

Screenshots and Screen Recording

const puppeteer = require('puppeteer');

async function screenshotExample() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  
  // Take a screenshot of the whole page
  await page.screenshot({ path: 'full-page.png', format: 'png' });
  
  // Take a screenshot of the visible area
  await page.screenshot({ path: 'visible-area.png', clip: { x: 0, y: 0, width: 1920, height: 1080 } });
  
  // Record a video
  await page.startRecordingVideo({ path: 'video.mp4' });
  await page.waitForNavigation();
  await page.stopRecordingVideo();
  
  await browser.close();
}
screenshotExample().catch(console.error);

✨ Features

• API Debugging: Supports communication via Chrome's debugging protocol.
• DOM Manipulation: Provides the ability to operate on DOM elements, including querying, clicking, and inputting.
• Page Actions: Can perform operations such as navigation, page refresh, and closing tabs.
• File Interaction: Supports reading local file content and injecting it into a new page.
• Screenshots and Screen Recording: Can generate screenshots or screen recording videos of web pages.

📦 Installation

npm install puppeteer

💻 Usage Examples

Web Scraping

const puppeteer = require('puppeteer');

async function webScraping() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  // Iterate through multiple pages
  for (let i = 1; i <= 5; i++) {
    await page.goto(`https://example.com?page=${i}`);
    const content = await page.textContent('.content');
    console.log(`Content of page ${i}: ${content}`);
  }
  
  await browser.close();
}
webScraping().catch(console.error);

Automated Form Submission

const puppeteer = require('puppeteer');

async function formAutomation() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com/form');
  
  // Fill out the form
  await page.type('#username', '测试用户');
  await page.type('#password', '测试密码');
  await page.click('button[type="submit"]');
  
  console.log('Form submitted successfully!');
  
  await browser.close();
}
formAutomation().catch(console.error);

📚 Documentation

Handling Asynchronous JavaScript

const puppeteer = require('puppeteer');

async function handleAsyncJs() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com/async');
  
  // Use page.waitFor to wait for a specific condition
  await page.waitFor(() => {
    return document.querySelectorAll('.loaded').length > 0;
  });
  
  console.log('Page loaded!');
  
  await browser.close();
}
handleAsyncJs().catch(console.error);

Handling Modal Dialogs

const puppeteer = require('puppeteer');

async function handleModals() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com/modal');
  
  // Listen for dialog events
  page.on('dialog', dialog => {
    console.log('Dialog detected:', dialog.message());
    dialog.accept();
  });
  
  await page.click('#open-modal');
  
  await browser.close();
}
handleModals().catch(console.error);

🔧 Best Practices

1. Handle Multiple Tabs: Create new tabs using browser.newPage() to avoid interference.
2. Use Waiting Methods: Always use page.waitFor before performing asynchronous operations to ensure element loading is complete.
3. Clean Up Resources: Remember to close the browser instance to release system resources.

📚 Project Examples

Example 1: Simple Web Page Screenshot

const puppeteer = require('puppeteer');

async function simpleScreenshot() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  
  await page.screenshot({ path: 'example-site.png' });
  
  await browser.close();
}
simpleScreenshot().catch(console.error);

Example 2: Automated Table Data Scraping

const puppeteer = require('puppeteer');

async function tableScraping() {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  for (let i = 1; i <= 3; i++) {
    await page.goto(`https://example.com/table?page=${i}`);
    
    // Get all table rows
    const rows = await page.querySelectorAll('tr');
    
    for (const row of rows) {
      console.log(await row.textContent());
    }
  }
  
  await browser.close();
}
tableScraping().catch(console.error);

Summary

Puppeteer is a powerful tool suitable for various browser automation tasks. Through its rich APIs, developers can easily implement complex functions such as web scraping, form submission, and page operations. Mastering Puppeteer will greatly improve development efficiency.

⚠️ Important Note

In practical applications, please comply with relevant laws, regulations, and website terms of use to avoid illegal or intrusive web scraping.