cURL
curl --request POST \
--url https://nano-gpt.com/api/scrape-urls \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"urls": [
"https://example.com/article",
"https://blog.com/post"
]
}
'import requests
url = "https://nano-gpt.com/api/scrape-urls"
payload = { "urls": ["https://example.com/article", "https://blog.com/post"] }
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({urls: ['https://example.com/article', 'https://blog.com/post']})
};
fetch('https://nano-gpt.com/api/scrape-urls', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://nano-gpt.com/api/scrape-urls",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'urls' => [
'https://example.com/article',
'https://blog.com/post'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://nano-gpt.com/api/scrape-urls"
payload := strings.NewReader("{\n \"urls\": [\n \"https://example.com/article\",\n \"https://blog.com/post\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://nano-gpt.com/api/scrape-urls")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"urls\": [\n \"https://example.com/article\",\n \"https://blog.com/post\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://nano-gpt.com/api/scrape-urls")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"urls\": [\n \"https://example.com/article\",\n \"https://blog.com/post\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"url": "<string>",
"success": true,
"title": "<string>",
"content": "<string>",
"markdown": "<string>",
"error": "<string>"
}
],
"summary": {
"requested": 123,
"processed": 123,
"successful": 123,
"failed": 123,
"totalCost": 123
}
}{
"error": 123,
"message": "<string>"
}{
"error": 123,
"message": "<string>"
}{
"error": 123,
"message": "<string>"
}Endpoint Examples
Web Scraping
Extract clean, formatted content from web pages. Returns both raw HTML content and formatted markdown.
POST
/
scrape-urls
cURL
curl --request POST \
--url https://nano-gpt.com/api/scrape-urls \
--header 'Content-Type: application/json' \
--header 'x-api-key: <api-key>' \
--data '
{
"urls": [
"https://example.com/article",
"https://blog.com/post"
]
}
'import requests
url = "https://nano-gpt.com/api/scrape-urls"
payload = { "urls": ["https://example.com/article", "https://blog.com/post"] }
headers = {
"x-api-key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'x-api-key': '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({urls: ['https://example.com/article', 'https://blog.com/post']})
};
fetch('https://nano-gpt.com/api/scrape-urls', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://nano-gpt.com/api/scrape-urls",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'urls' => [
'https://example.com/article',
'https://blog.com/post'
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"x-api-key: <api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://nano-gpt.com/api/scrape-urls"
payload := strings.NewReader("{\n \"urls\": [\n \"https://example.com/article\",\n \"https://blog.com/post\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("x-api-key", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://nano-gpt.com/api/scrape-urls")
.header("x-api-key", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"urls\": [\n \"https://example.com/article\",\n \"https://blog.com/post\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://nano-gpt.com/api/scrape-urls")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["x-api-key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"urls\": [\n \"https://example.com/article\",\n \"https://blog.com/post\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"url": "<string>",
"success": true,
"title": "<string>",
"content": "<string>",
"markdown": "<string>",
"error": "<string>"
}
],
"summary": {
"requested": 123,
"processed": 123,
"successful": 123,
"failed": 123,
"totalCost": 123
}
}{
"error": 123,
"message": "<string>"
}{
"error": 123,
"message": "<string>"
}{
"error": 123,
"message": "<string>"
}Overview
The NanoGPT Web Scraping API allows you to extract clean, formatted content from web pages. It uses the Firecrawl service to scrape URLs and returns both raw HTML content and formatted markdown. You can also call this tool through the unified Data API atPOST /api/v1/data/url/scrape. The Data API preserves this endpoint’s request body, response body, billing, and validation behavior while adding discovery and dispatch metadata.
Authentication
The API supports two authentication methods:1. API Key Authentication (Recommended)
Include your API key in the request header:x-api-key: YOUR_API_KEY
2. Bearer Token Authentication
Authorization: Bearer YOUR_API_KEY
Accountless x402 Payment
For accountless payment, prefer the public Data API path:curl -i https://nano-gpt.com/api/v1/data/url/scrape \
-H "Content-Type: application/json" \
-H "x-x402: true" \
-d '{
"urls": ["https://nano-gpt.com"],
"stealthMode": false
}'
Authorization or x-api-key, and include x-x402: true. NanoGPT will return 402 Payment Required with available payment options. This endpoint supports accountless x402 payments where listed by GET /api/v1/x402/endpoints, including Lightning L402 when advertised. See Accountless x402 API Payments for the full flow.
If you receive 401 missing_api_key immediately, check that the initial quote request includes x-x402: true. Without that header, NanoGPT does not enter the x402 quote flow.
Request Format
Headers
Content-Type: application/json
x-api-key: YOUR_API_KEY
Request Body
{
"urls": [
"https://example.com/article",
"https://blog.com/post",
"https://news.site.com/story"
],
"stealthMode": false
}
Parameters
| Parameter | Type | Required | Description |
|---|---|---|---|
| urls | string[] | Yes | Array of URLs to scrape. Maximum 5 URLs per request. |
| stealthMode | boolean | No | Optional. Default false. When true, multiplies the upfront per-URL charge by 5 and routes requests through the stealth proxy. |
Stealth scraping (optional)
SetstealthMode: true to run requests through Firecrawl’s stealth proxy for tougher targets. Stealth scraping costs 5× the standard per-URL rate and still counts toward the configured URL cap. The web UI exposes the same toggle, so use this field to mirror that behavior from the API.
POST /api/scrape-urls
{
"urls": ["https://example.com/restricted"],
"stealthMode": true
}
summary includes stealthModeUsed so you can track when the surcharge applied.
URL Requirements
- Must be valid HTTP or HTTPS URLs
- Must have standard web ports (80, 443, or default)
- Cannot be localhost, private IPs, or metadata endpoints
- YouTube URLs are not supported (use the YouTube transcription endpoint instead)
Response Format
Success Response (200 OK)
{
"results": [
{
"url": "https://example.com/article",
"success": true,
"title": "Article Title",
"content": "Raw HTML content...",
"markdown": "# Article Title\n\nFormatted markdown content..."
},
{
"url": "https://invalid.site.com",
"success": false,
"error": "Failed to scrape URL"
}
],
"summary": {
"requested": 3,
"processed": 3,
"successful": 2,
"failed": 1,
"totalCost": 0.002,
"stealthModeUsed": false
}
}
Response Fields
results
Array of scraping results for each URL:url(string): The URL that was scrapedsuccess(boolean): Whether the scraping was successfultitle(string, optional): Page title if successfully scrapedcontent(string, optional): Raw HTML contentmarkdown(string, optional): Formatted markdown version of the contenterror(string, optional): Error message if scraping failed
summary
Summary statistics for the request:requested(number): Number of URLs in the original requestprocessed(number): Number of valid URLs that were processedsuccessful(number): Number of URLs successfully scrapedfailed(number): Number of URLs that failed to scrapetotalCost(number): Total cost in USD (only for successful scrapes)stealthModeUsed(boolean): Indicates whether stealth mode was enabled for any processed URLs
Error Responses
400 Bad Request
{
"error": "Please provide an array of URLs to scrape"
}
401 Unauthorized
{
"error": "Invalid session"
}
402 Payment Required
{
"error": "Insufficient balance"
}
429 Too Many Requests
{
"error": "Rate limit exceeded. Please wait before sending another request."
}
500 Internal Server Error
{
"error": "An error occurred while processing your request"
}
Pricing
- Cost: $0.001 per successfully scraped URL
- Billing: You are only charged for URLs that are successfully scraped
- Payment Methods: USD balance or Nano (XNO) cryptocurrency
- Note (inline chat scraping): Inline URL scraping inside
POST /api/v1/chat/completions(viascraping: true) is billed separately at $0.0015 per successfully scraped URL. Use this endpoint when you want explicit URL lists and the lower standalone price.
Rate Limits
- Default: 30 requests per minute per IP address
- With API Key: 30 requests per minute per API key
Code Examples
curl -X POST https://nano-gpt.com/api/scrape-urls \
-H "Content-Type: application/json" \
-H "x-api-key: YOUR_API_KEY" \
-d '{
"urls": [
"https://example.com/article",
"https://blog.com/post"
]
}'
import requests
api_key = "YOUR_API_KEY"
urls = [
"https://example.com/article",
"https://blog.com/post"
]
response = requests.post(
"https://nano-gpt.com/api/scrape-urls",
headers={
"Content-Type": "application/json",
"x-api-key": api_key
},
json={"urls": urls}
)
data = response.json()
for result in data["results"]:
if result["success"]:
print(f"Title: {result['title']}")
print(f"Markdown: {result['markdown'][:200]}...")
else:
print(f"Failed to scrape {result['url']}: {result['error']}")
const apiKey = 'YOUR_API_KEY';
const urls = [
'https://example.com/article',
'https://blog.com/post'
];
const response = await fetch('https://nano-gpt.com/api/scrape-urls', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': apiKey
},
body: JSON.stringify({ urls })
});
const data = await response.json();
data.results.forEach(result => {
if (result.success) {
console.log(`Title: ${result.title}`);
console.log(`Markdown: ${result.markdown.substring(0, 200)}...`);
} else {
console.log(`Failed to scrape ${result.url}: ${result.error}`);
}
});
Best Practices
- Batch Requests: Send multiple URLs in a single request (up to 5) to minimize API calls
- Error Handling: Always check the
successfield for each result before accessing content - Content Size: Scraped content is limited to 100KB per URL
- URL Validation: Validate URLs on your end before sending to reduce failed requests
- Markdown Format: Use the markdown field for better readability and formatting
Limitations
- Maximum 5 URLs per request
- Maximum content size: 100KB per URL
- No JavaScript rendering (static content only)
FAQ
Q: Why was my URL rejected? A: URLs can be rejected for several reasons:- Invalid format (not HTTP/HTTPS)
- Pointing to localhost or private IPs
- Using non-standard ports
- Being a YouTube URL (use the YouTube transcription endpoint)
Authorizations
apiKeyAuthbearerAuth
Body
application/json
Web scraping parameters
Array of URLs to scrape (maximum 5 URLs per request)
Required array length:
1 - 5 elementsURL to scrape
Example:
[
"https://example.com/article",
"https://blog.com/post"
]