JavaScript
import { Hyperbrowser } from '@hyperbrowser/sdk';
const client = new Hyperbrowser({ apiKey: 'your-api-key' });
const response = await client.web.crawl.start({
url: 'https://example.com',
outputs: {
formats: ['markdown']
},
crawlOptions: {
maxPages: 10,
followLinks: true
}
});from hyperbrowser import Hyperbrowser
from hyperbrowser.models import StartWebCrawlJobParams, WebCrawlOptions, FetchOutputOptions
client = Hyperbrowser(api_key='your-api-key')
response = client.web.crawl.start(StartWebCrawlJobParams(
url='https://example.com',
outputs=FetchOutputOptions(
formats=['markdown']
),
crawl_options=WebCrawlOptions(
max_pages=10,
follow_links=True
)
))curl --request POST \
--url https://api.hyperbrowser.ai/api/web/crawl \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"url": "<string>",
"outputs": {
"formats": [
{
"type": "markdown"
}
],
"includeSelectors": [
"<string>"
],
"excludeSelectors": [
"<string>"
],
"storageState": {
"localStorage": {},
"sessionStorage": {}
}
},
"browser": {
"screen": {
"width": 1280,
"height": 720
},
"profileId": "<string>",
"solveCaptchas": "<string>",
"location": {
"country": "<string>",
"state": "<string>",
"city": "<string>"
}
},
"navigation": {
"timeoutMs": 123,
"waitFor": 123
},
"cache": {
"maxAgeSeconds": 123
},
"crawlOptions": {
"maxPages": 10,
"followLinks": true,
"ignoreSitemap": false,
"excludePatterns": [
"<string>"
],
"includePatterns": [
"<string>"
]
}
}
'<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.hyperbrowser.ai/api/web/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'outputs' => [
'formats' => [
[
'type' => 'markdown'
]
],
'includeSelectors' => [
'<string>'
],
'excludeSelectors' => [
'<string>'
],
'storageState' => [
'localStorage' => [
],
'sessionStorage' => [
]
]
],
'browser' => [
'screen' => [
'width' => 1280,
'height' => 720
],
'profileId' => '<string>',
'solveCaptchas' => '<string>',
'location' => [
'country' => '<string>',
'state' => '<string>',
'city' => '<string>'
]
],
'navigation' => [
'timeoutMs' => 123,
'waitFor' => 123
],
'cache' => [
'maxAgeSeconds' => 123
],
'crawlOptions' => [
'maxPages' => 10,
'followLinks' => true,
'ignoreSitemap' => false,
'excludePatterns' => [
'<string>'
],
'includePatterns' => [
'<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.hyperbrowser.ai/api/web/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"outputs\": {\n \"formats\": [\n {\n \"type\": \"markdown\"\n }\n ],\n \"includeSelectors\": [\n \"<string>\"\n ],\n \"excludeSelectors\": [\n \"<string>\"\n ],\n \"storageState\": {\n \"localStorage\": {},\n \"sessionStorage\": {}\n }\n },\n \"browser\": {\n \"screen\": {\n \"width\": 1280,\n \"height\": 720\n },\n \"profileId\": \"<string>\",\n \"solveCaptchas\": \"<string>\",\n \"location\": {\n \"country\": \"<string>\",\n \"state\": \"<string>\",\n \"city\": \"<string>\"\n }\n },\n \"navigation\": {\n \"timeoutMs\": 123,\n \"waitFor\": 123\n },\n \"cache\": {\n \"maxAgeSeconds\": 123\n },\n \"crawlOptions\": {\n \"maxPages\": 10,\n \"followLinks\": true,\n \"ignoreSitemap\": false,\n \"excludePatterns\": [\n \"<string>\"\n ],\n \"includePatterns\": [\n \"<string>\"\n ]\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.hyperbrowser.ai/api/web/crawl")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"outputs\": {\n \"formats\": [\n {\n \"type\": \"markdown\"\n }\n ],\n \"includeSelectors\": [\n \"<string>\"\n ],\n \"excludeSelectors\": [\n \"<string>\"\n ],\n \"storageState\": {\n \"localStorage\": {},\n \"sessionStorage\": {}\n }\n },\n \"browser\": {\n \"screen\": {\n \"width\": 1280,\n \"height\": 720\n },\n \"profileId\": \"<string>\",\n \"solveCaptchas\": \"<string>\",\n \"location\": {\n \"country\": \"<string>\",\n \"state\": \"<string>\",\n \"city\": \"<string>\"\n }\n },\n \"navigation\": {\n \"timeoutMs\": 123,\n \"waitFor\": 123\n },\n \"cache\": {\n \"maxAgeSeconds\": 123\n },\n \"crawlOptions\": {\n \"maxPages\": 10,\n \"followLinks\": true,\n \"ignoreSitemap\": false,\n \"excludePatterns\": [\n \"<string>\"\n ],\n \"includePatterns\": [\n \"<string>\"\n ]\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.hyperbrowser.ai/api/web/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"outputs\": {\n \"formats\": [\n {\n \"type\": \"markdown\"\n }\n ],\n \"includeSelectors\": [\n \"<string>\"\n ],\n \"excludeSelectors\": [\n \"<string>\"\n ],\n \"storageState\": {\n \"localStorage\": {},\n \"sessionStorage\": {}\n }\n },\n \"browser\": {\n \"screen\": {\n \"width\": 1280,\n \"height\": 720\n },\n \"profileId\": \"<string>\",\n \"solveCaptchas\": \"<string>\",\n \"location\": {\n \"country\": \"<string>\",\n \"state\": \"<string>\",\n \"city\": \"<string>\"\n }\n },\n \"navigation\": {\n \"timeoutMs\": 123,\n \"waitFor\": 123\n },\n \"cache\": {\n \"maxAgeSeconds\": 123\n },\n \"crawlOptions\": {\n \"maxPages\": 10,\n \"followLinks\": true,\n \"ignoreSitemap\": false,\n \"excludePatterns\": [\n \"<string>\"\n ],\n \"includePatterns\": [\n \"<string>\"\n ]\n }\n}"
response = http.request(request)
puts response.read_body{
"jobId": "<string>"
}{
"message": "<string>"
}{
"message": "<string>"
}Web API
Start a web crawl job
Starts an asynchronous crawl job that follows links from a starting URL and returns content from each page in the specified formats.
POST
/
api
/
web
/
crawl
JavaScript
import { Hyperbrowser } from '@hyperbrowser/sdk';
const client = new Hyperbrowser({ apiKey: 'your-api-key' });
const response = await client.web.crawl.start({
url: 'https://example.com',
outputs: {
formats: ['markdown']
},
crawlOptions: {
maxPages: 10,
followLinks: true
}
});from hyperbrowser import Hyperbrowser
from hyperbrowser.models import StartWebCrawlJobParams, WebCrawlOptions, FetchOutputOptions
client = Hyperbrowser(api_key='your-api-key')
response = client.web.crawl.start(StartWebCrawlJobParams(
url='https://example.com',
outputs=FetchOutputOptions(
formats=['markdown']
),
crawl_options=WebCrawlOptions(
max_pages=10,
follow_links=True
)
))curl --request POST \
--url https://api.hyperbrowser.ai/api/web/crawl \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"url": "<string>",
"outputs": {
"formats": [
{
"type": "markdown"
}
],
"includeSelectors": [
"<string>"
],
"excludeSelectors": [
"<string>"
],
"storageState": {
"localStorage": {},
"sessionStorage": {}
}
},
"browser": {
"screen": {
"width": 1280,
"height": 720
},
"profileId": "<string>",
"solveCaptchas": "<string>",
"location": {
"country": "<string>",
"state": "<string>",
"city": "<string>"
}
},
"navigation": {
"timeoutMs": 123,
"waitFor": 123
},
"cache": {
"maxAgeSeconds": 123
},
"crawlOptions": {
"maxPages": 10,
"followLinks": true,
"ignoreSitemap": false,
"excludePatterns": [
"<string>"
],
"includePatterns": [
"<string>"
]
}
}
'<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.hyperbrowser.ai/api/web/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'outputs' => [
'formats' => [
[
'type' => 'markdown'
]
],
'includeSelectors' => [
'<string>'
],
'excludeSelectors' => [
'<string>'
],
'storageState' => [
'localStorage' => [
],
'sessionStorage' => [
]
]
],
'browser' => [
'screen' => [
'width' => 1280,
'height' => 720
],
'profileId' => '<string>',
'solveCaptchas' => '<string>',
'location' => [
'country' => '<string>',
'state' => '<string>',
'city' => '<string>'
]
],
'navigation' => [
'timeoutMs' => 123,
'waitFor' => 123
],
'cache' => [
'maxAgeSeconds' => 123
],
'crawlOptions' => [
'maxPages' => 10,
'followLinks' => true,
'ignoreSitemap' => false,
'excludePatterns' => [
'<string>'
],
'includePatterns' => [
'<string>'
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.hyperbrowser.ai/api/web/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"outputs\": {\n \"formats\": [\n {\n \"type\": \"markdown\"\n }\n ],\n \"includeSelectors\": [\n \"<string>\"\n ],\n \"excludeSelectors\": [\n \"<string>\"\n ],\n \"storageState\": {\n \"localStorage\": {},\n \"sessionStorage\": {}\n }\n },\n \"browser\": {\n \"screen\": {\n \"width\": 1280,\n \"height\": 720\n },\n \"profileId\": \"<string>\",\n \"solveCaptchas\": \"<string>\",\n \"location\": {\n \"country\": \"<string>\",\n \"state\": \"<string>\",\n \"city\": \"<string>\"\n }\n },\n \"navigation\": {\n \"timeoutMs\": 123,\n \"waitFor\": 123\n },\n \"cache\": {\n \"maxAgeSeconds\": 123\n },\n \"crawlOptions\": {\n \"maxPages\": 10,\n \"followLinks\": true,\n \"ignoreSitemap\": false,\n \"excludePatterns\": [\n \"<string>\"\n ],\n \"includePatterns\": [\n \"<string>\"\n ]\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.hyperbrowser.ai/api/web/crawl")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"outputs\": {\n \"formats\": [\n {\n \"type\": \"markdown\"\n }\n ],\n \"includeSelectors\": [\n \"<string>\"\n ],\n \"excludeSelectors\": [\n \"<string>\"\n ],\n \"storageState\": {\n \"localStorage\": {},\n \"sessionStorage\": {}\n }\n },\n \"browser\": {\n \"screen\": {\n \"width\": 1280,\n \"height\": 720\n },\n \"profileId\": \"<string>\",\n \"solveCaptchas\": \"<string>\",\n \"location\": {\n \"country\": \"<string>\",\n \"state\": \"<string>\",\n \"city\": \"<string>\"\n }\n },\n \"navigation\": {\n \"timeoutMs\": 123,\n \"waitFor\": 123\n },\n \"cache\": {\n \"maxAgeSeconds\": 123\n },\n \"crawlOptions\": {\n \"maxPages\": 10,\n \"followLinks\": true,\n \"ignoreSitemap\": false,\n \"excludePatterns\": [\n \"<string>\"\n ],\n \"includePatterns\": [\n \"<string>\"\n ]\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.hyperbrowser.ai/api/web/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"outputs\": {\n \"formats\": [\n {\n \"type\": \"markdown\"\n }\n ],\n \"includeSelectors\": [\n \"<string>\"\n ],\n \"excludeSelectors\": [\n \"<string>\"\n ],\n \"storageState\": {\n \"localStorage\": {},\n \"sessionStorage\": {}\n }\n },\n \"browser\": {\n \"screen\": {\n \"width\": 1280,\n \"height\": 720\n },\n \"profileId\": \"<string>\",\n \"solveCaptchas\": \"<string>\",\n \"location\": {\n \"country\": \"<string>\",\n \"state\": \"<string>\",\n \"city\": \"<string>\"\n }\n },\n \"navigation\": {\n \"timeoutMs\": 123,\n \"waitFor\": 123\n },\n \"cache\": {\n \"maxAgeSeconds\": 123\n },\n \"crawlOptions\": {\n \"maxPages\": 10,\n \"followLinks\": true,\n \"ignoreSitemap\": false,\n \"excludePatterns\": [\n \"<string>\"\n ],\n \"includePatterns\": [\n \"<string>\"\n ]\n }\n}"
response = http.request(request)
puts response.read_body{
"jobId": "<string>"
}{
"message": "<string>"
}{
"message": "<string>"
}Authorizations
Body
application/json
Response
Crawl job started successfully
Was this page helpful?
⌘I