Веб-кроулинг с помощью Java - параллельный кроулинг, безголовый браузер и браузер для защиты от несанкционированного доступа

import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;
import java.io.IOException;
import java.util.*;

public class Scraper {
  /**
first page of scrapeme products list
   */
  private static final String SCRAPEME_SITE_URL = "https://scrapeme.live/shop";

  public static void scrape(List<ScrapeMeProduct> scrapeMeProducts, Set<String> pagesFound, List<String> todoPages) {
    // html doc for scrapeme page
    Document doc;
    // remove page from todoPages
    String url = todoPages.removeFirst();
    try {
      doc = Jsoup.connect(url).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36").header("Accept-Language", "*").get();
      // select product nodes
      Elements products = doc.select("li.product");
      for (Element product : products) {
        ScrapeMeProduct scrapeMeProduct = new ScrapeMeProduct();
        scrapeMeProduct.setUrl(product.selectFirst("a").attr("href")); // parse and set product url
        scrapeMeProduct.setImage(product.selectFirst("img").attr("src")); // parse and set product image
        scrapeMeProduct.setName(product.selectFirst("h2").text()); // parse and set product name
        scrapeMeProduct.setPrice(product.selectFirst("span").text()); // parse and set product price
        scrapeMeProducts.add(scrapeMeProduct);
      }
      // add to pages found set
      pagesFound.add(url);
      Elements paginationElements = doc.select("a.page-numbers");
      for (Element pageElement : paginationElements) {
        String pageUrl = pageElement.attr("href");
        // add new pages to todoPages
        if (!pagesFound.contains(pageUrl) && !todoPages.contains(pageUrl)) {
          todoPages.add(pageUrl);
        }
        // add to pages found set
        pagesFound.add(pageUrl);
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  public static List<ScrapeMeProduct> scrapeAll() {
    // products
    List<ScrapeMeProduct> scrapeMeProducts = new ArrayList<>();
    // all pages found
    Set<String> pagesFound = new HashSet<>();
    // pages list waiting for scrape
    List<String> todoPages = new ArrayList<>();
    // add the first page to scrape
    todoPages.add(SCRAPEME_SITE_URL);
    while (!todoPages.isEmpty()) {
      scrape(scrapeMeProducts, pagesFound, todoPages);
    }
    return scrapeMeProducts;
  }
}

import io.xxx.basic.ScrapeMeProduct;
import io.xxx.basic.Scraper;
import java.util.List;

public class Main {
  public static void main(String[] args) {
    List<ScrapeMeProduct> products = Scraper.scrapeAll();
    System.out.println(products.size() + " products scraped");
    // then you can do whatever you want
  }
}

//  duplicates omitted
public static void concurrentScrape() {
  // using synchronized collections
  List<ScrapeMeProduct> pokemonProducts = Collections.synchronizedList(new ArrayList<>());
  Set<String> pagesDiscovered = Collections.synchronizedSet(new HashSet<>());
  List<String> pagesToScrape = Collections.synchronizedList(new ArrayList<>());
  pagesToScrape.add(SCRAPEME_SITE_URL);
  // new thread pool with CPU cores
  ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
  scrape(pokemonProducts, pagesDiscovered, pagesToScrape);
  try {
    while (!pagesToScrape.isEmpty()) {
      executorService.execute(() -> scrape(pokemonProducts, pagesDiscovered, pagesToScrape));
      // sleep for a while for all pending threads to end
      TimeUnit.MILLISECONDS.sleep(300);
    }
    executorService.shutdown();
    executorService.awaitTermination(5, TimeUnit.MINUTES);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}

// gradle => build.gradle => dependencies
implementation "org.seleniumhq.selenium:selenium-java:4.14.1"

import io.xxx.basic.ScrapeMeProduct;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

public class NstbrowserScraper {
  // scrapeme site url
  private static final String SCRAPEME_SITE_URL = "https://scrapeme.live/shop";
  // Nstbrowser LaunchNewBrowser api url
  private static final String NSTBROWSER_LAUNCH_BROWSER_API = "http://127.0.0.1:8848/api/agent/devtool/launch";

  /**
   * Launches a new browser instance using the Nstbrowser LaunchNewBrowser API.
   */
  public static void launchBrowser(String port) throws Exception {
    String config = buildLaunchNewBrowserQueryConfig(port);
    String launchUrl = NSTBROWSER_LAUNCH_BROWSER_API + "?config=" + config;
    URL url = new URL(launchUrl);
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    conn.setRequestMethod("GET");
    // set request headers
    conn.setRequestProperty("User-Agent", "Mozilla/5.0");
    conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
    conn.setRequestProperty("x-api-key", "your Nstbrowser api key");
    conn.setDoOutput(true);
    try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
      String inputLine;
      StringBuilder response = new StringBuilder();
      while ((inputLine = in.readLine()) != null) {
        response.append(inputLine);
      }
      // deal with response ...
    }
  }

  /**
   * Builds the JSON configuration for launching a new browser instance.
   *
   */
  private static String buildLaunchNewBrowserQueryConfig(String port) {
    String jsonParam = """
        {
            "once": true,
            "headless": false,
            "autoClose": false,
            "remoteDebuggingPort": %port,
            "fingerprint": {
                "name": "test",
                "kernel": "chromium",
                "platform": "mac",
                "kernelMilestone": "120",
                "hardwareConcurrency": 10,
                "deviceMemory": 8
            }
        }
        """;
    jsonParam = jsonParam.replace("%port", port);
    return URLEncoder.encode(jsonParam, StandardCharsets.UTF_8);
  }

  /**
   * Scrapes product data from the Scrapeme website using Nstbrowser headless browser.
   *
   */
  public static List<ScrapeMeProduct> scrape(String port) {
    ChromeOptions options = new ChromeOptions();
    // enable headless mode
    options.addArguments("--headless");
    // set driver path
    System.setProperty("webdriver.chrome.driver", "your chrome webdriver path");
    System.setProperty("webdriver.http.factory", "jdk-http-client");
    // create options
    // debuggerAddress
    options.setExperimentalOption("debuggerAddress", "127.0.0.1:" + port);
    options.addArguments("--remote-allow-origins=*");
    WebDriver driver = new ChromeDriver(options);
    driver.get(SCRAPEME_SITE_URL);
    // products data
    List<ScrapeMeProduct> pokemonProducts = new ArrayList<>();
    List<WebElement> products = driver.findElements(By.cssSelector("li.product"));
    for (WebElement product : products) {
      ScrapeMeProduct pokemonProduct = new ScrapeMeProduct();
      pokemonProduct.setUrl(product.findElement(By.tagName("a")).getAttribute("href")); // parse and set product url
      pokemonProduct.setImage(product.findElement(By.tagName(("img"))).getAttribute("src")); // parse and set product image
      pokemonProduct.setName(product.findElement(By.tagName(("h2"))).getText()); // parse and set product name
      pokemonProduct.setPrice(product.findElement(By.tagName(("span"))).getText()); // parse and set product price
      pokemonProducts.add(pokemonProduct);
    }
    // quit browser
    driver.quit();
    return pokemonProducts;
  }

  public static void main(String[] args) {
    // browser remote debug port
    String port = "9222";
    try {
      launchBrowser(port);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
    List<ScrapeMeProduct> products = scrape(port);
    products.forEach(System.out::println);
  }
}