RAG爬虫太拉垮？快来试试智能爬虫Crawl4AI，开源高效，专为AI量身打造！附实测效果 原创

pip3 install crawl4ai # 安装 crawl4ai 库

crawl4ai-setup # 设置浏览器

# crawl4ai-doctor

[INIT].... → Running Crawl4AI health check...
[INIT].... → Crawl4AI 0.5.0.post2
[TEST].... ℹ Testing crawling capabilities...

[EXPORT].. ℹ Exporting PDF and taking screenshot took 1.31s
[FETCH]... ↓ https://crawl4ai.com... | Status: True | Time: 4.04s
[SCRAPE].. ◆ https://crawl4ai.com... | Time: 0.06s
[COMPLETE] ● https://crawl4ai.com... | Status: True | Total: 4.10s
[COMPLETE] ● ✅ Crawling test passed!

python -m playwright install --with-deps chromium

import asyncio
from crawl4ai import *

asyncdef main():
    asyncwith AsyncWebCrawler() as crawler:
        run_config = CrawlerRunConfig(
            word_count_threshold=10,  # 每个内容块的最小字数
            exclude_external_links=True,  # 移除外部链接
            remove_overlay_elements=True,  # 移除弹窗/模态框
            process_iframes=True# 处理iframe内容
        )
        result = await crawler.arun(
            url="https://www.sentosa.com.sg/en/get-inspired/sentosa-guides/top-free-things-to-do",
            config=run_config
        )
        print(result.markdown)

if __name__ == "__main__":
    asyncio.run(main())

import asyncio
from crawl4ai import *
from pydantic import BaseModel, Field

INSTRUCTION_TO_LLM = """从抓取的内容中提取所有的标题，内容和标题的图片链接link"""

class Sentosa(BaseModel):
    name: str = Field(..., descriptinotallow="标题")
    content: str = Field(..., descriptinotallow="内容")
    link: str = Field(..., descriptinotallow="链接link")


llm_strategy = LLMExtractionStrategy(
        llm_cnotallow=LLMConfig(provider="openai/gpt-4o", api_token="api_key"),
        schema=Sentosa.model_json_schema(),
        extraction_type="schema",
        instructinotallow=INSTRUCTION_TO_LLM,
        chunk_token_threshold=1000,
        overlap_rate=0.0,
        apply_chunking=True,
        input_format="markdown",
        extra_args={"temperature": 0.0, "max_tokens": 3600},
    )

browser_cfg = BrowserConfig(headless=True, verbose=True)

asyncdef main():
    asyncwith AsyncWebCrawler(cnotallow=browser_cfg) as crawler:
        run_config = CrawlerRunConfig(
            word_count_threshold=10,  # Minimum words per content block
            exclude_external_links=True,  # Remove external links
            remove_overlay_elements=True,  # Remove popups/modals
            process_iframes=True,  # Process iframe content,
            extraction_strategy=llm_strategy
        )

        result = await crawler.arun(
            url="https://www.sentosa.com.sg/en/get-inspired/sentosa-guides/top-free-things-to-do",
            cnotallow=run_config
        )
        print(result.extracted_content)

if __name__ == "__main__":
    asyncio.run(main())

[
    {
        "name": "Top Free Things to Do in Sentosa",
        "content": "Explore the best free activities and attractions in Sentosa, from beautiful beaches to scenic nature trails.",
        "link": "https://www.sentosa.com.sg/en/get-inspired/sentosa-guides/top-free-things-to-do",
        "error": false
    },
    {
        "name": "Stroll along the Sentosa Boardwalk",
        "content": "Before we get to do all the exciting stuff waiting at Sentosa, we have to get there first. And what better way to do that than to stroll along the Sentosa Boardwalk? With its picturesque view of the city backdrop across the sea, it’ll be a waste not to snap a picture and share it with all your friends to see!",
        "link": "https://www.sentosa.com.sg/-/media/sentosa/article-listing/articles/13-free-things-to-do/13tipsgallery14.jpg?revisinotallow=622c8081-1f61-427f-a8f1-e4da1d457ca6",
        "error": false
    },
    {
        "name": "Chill at Tanjong Beach",
        "content": "For those looking to have a relaxing time at the beach indulging in their favourite book and music, Tanjong Beach is definitely the place to get away from the hectic pace of city life. Its tranquil atmosphere is thanks to its remote location at the southern end of Sentosa beachfront coastline.",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/tanjong-beach/",
        "error": false
    },
    {
        "name": "Explore the southernmost tip of Asia",
        "content": "For some adventuring at the beach, cross the suspension bridge at Palawan Beach to the southernmost tip of Asia. While you’re there, be sure to climb up the watchtower to enjoy a panoramic view of the South China sea as ships sail pass.",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/southernmost-point-of-continental-asia/",
        "error": false
    },
    {
        "name": "Play some beach sports at Siloso Beach",
        "content": "The beach isn’t just for sightseeing and chilling, it’s where all sorts of people gather to play beach sports such as Beach Volleyball, Ultimate Frisbee and Football! So call up your friends and family and head down to Siloso Beach to engage in one of the most fun beach activities you can do!",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/siloso-beach/",
        "error": false
    },
    {
        "name": "Unleash and discover magical experiences at Sensoryscape",
        "content": "Calling all my nature lovers, those who love scenic views or mesmerising interactive projections! Sensoryscape is the perfect place for you, and it's completely free too! So come on down for relaxing vibes during the day and stay for an enchanting night experience. As night falls, watch how this calming place transforms into the ImagiNite experience. Immerse yourself in the various sensory gardens like Symphony Streams enchanting underwater world, interactive projections at Palate Playground, dancing light beams at Lookout Loop, glowing giant flower stalks at Glow Garden and more. Don’t forget to download the ImagiNite App and witness the light shows and projections come to life! End your day in the most magical way possible at Sensoryscape where one can experience a blooming merge between vibrant reefs and lush ridges.",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/sensoryscape/",
        "error": false
    },
    {
        "name": "Walk along Fort Siloso Skywalk",
        "content": "Singapore has its own fair share of well-known Skywalks such as the OCBC Skyway and Henderson Waves, but the tallest one yet is Fort Siloso Skywalk, at 11 storeys high! It boasts beautiful views of Western Sentosa, Mount Faber and Keppel Harbour. Be sure to take plenty of photos but please don’t drop your phones while doing so!",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/fort-siloso-skywalk/",
        "error": false
    },
    {
        "name": "Go back in time at Fort Siloso",
        "content": "Singapore’s only preserved coastal fort is a treasure trove of WWII memorabilia. You can learn of its rich history by walking along its two trails: the Heritage Trail and the Gun Trail. Alternatively, you can go back in time with the Surrender Chambers immersive show where you get to relive Singapore’s epoch-making events.",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/fort-siloso/",
        "error": false
    },
    {
        "name": "Go hiking at Sentosa Nature Discovery",
        "content": "Nature lovers are sure to love this particular activity because in this 1.8-kilometre trek through a rainforest, you’ll get to get up and close with the birds, insects, plants of Sentosa. If you’re really observant, there’s over 20 different species of birds and even other animals like geckos and squirrels to see!",
        "link": "https://www.sentosa.com.sg/en/things-to-do/attractions/sentosa-nature-discovery/",
        "error": false
    },
    {
        "name": "Bring your date to Quayside Isle",
        "content": "Situated near Sentosa Cove, taking a leisurely walk along Quayside Isle’s cobbled pavements in the evening is the perfect way to wrap up a date. The neatly arranged yachts also lend some charm to the scene of the setting sun. With its quiet atmosphere and scenic view, it’s a great way to unwind and end the day.",
        "link": "",
        "error": false
    },
    {
        "name": "Experience free live music & events",
        "content": "From outdoor movie nights to beach concerts, Sentosa is always alive with free entertainment. Keep an eye on Sentosa’s event calendar for upcoming music gigs, pop-up markets, and cultural performances.",
        "link": "https://www.sentosa.com.sg/en/things-to-do/events/live-music-performance/",
        "error": false
    },
    {
        "name": "Hike through Sentosa’s nature trails",
        "content": "Escape the crowds and explore Sentosa’s lush greenery with these hidden gems: Imbiah Trail – Spot unique wildlife and discover ancient rock formations. Coastal Trail – A scenic, seaside route with panoramic ocean views.",
        "link": "",
        "error": false
    }
]

async with AsyncWebCrawler() as crawler:
    result = await crawler.arun(
        url="https://example.com",
        js_code="window.scrollTo(0, document.body.scrollHeight);",
        wait_for="document.querySelector('.content-loaded')"
    )
    print(result.markdown)