65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
import asyncio
|
|
from playwright.async_api import async_playwright
|
|
import json
|
|
|
|
async def scrape_page():
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=True)
|
|
page = await browser.new_page()
|
|
|
|
await page.goto('https://meetup.hackrobot.cn/salon', wait_until='networkidle', timeout=30000)
|
|
|
|
# 等待页面加载完成
|
|
await page.wait_for_timeout(3000)
|
|
|
|
# 获取页面完整HTML
|
|
content = await page.content()
|
|
|
|
# 保存HTML
|
|
with open('salon_full.html', 'w', encoding='utf-8') as f:
|
|
f.write(content)
|
|
|
|
# 获取页面标题
|
|
title = await page.title()
|
|
print(f"页面标题: {title}")
|
|
|
|
# 获取页面的视口高度
|
|
viewport_height = await page.evaluate('window.innerHeight')
|
|
page_height = await page.evaluate('document.body.scrollHeight')
|
|
|
|
print(f"视口高度: {viewport_height}, 页面总高度: {page_height}")
|
|
|
|
# 滚动并截图
|
|
screenshots = []
|
|
scroll_position = 0
|
|
screenshot_count = 0
|
|
|
|
while scroll_position < page_height:
|
|
await page.screenshot(path=f'screenshot_{screenshot_count}.png', full_page=False)
|
|
screenshots.append(f'screenshot_{screenshot_count}.png')
|
|
screenshot_count += 1
|
|
|
|
scroll_position += viewport_height
|
|
await page.evaluate(f'window.scrollTo(0, {scroll_position})')
|
|
await page.wait_for_timeout(1000)
|
|
|
|
# 最后截取完整页面
|
|
await page.screenshot(path='screenshot_full.png', full_page=True)
|
|
|
|
print(f"截图完成: 共 {screenshot_count} 张分段截图 + 1张完整截图")
|
|
|
|
# 获取所有文本内容
|
|
text_content = await page.evaluate('''() => {
|
|
return document.body.innerText;
|
|
}''')
|
|
|
|
with open('page_text.txt', 'w', encoding='utf-8') as f:
|
|
f.write(text_content)
|
|
|
|
print("文本内容已保存")
|
|
|
|
await browser.close()
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(scrape_page())
|