's'

2026-03-07 09:34:01 -06:00
parent 742815f2db
commit da5f9c18a7
39 changed files with 8602 additions and 0 deletions
--- a/scrape_salon.py
+++ b/scrape_salon.py
@@ -0,0 +1,64 @@
+import asyncio
+from playwright.async_api import async_playwright
+import json
+
+async def scrape_page():
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        page = await browser.new_page()
+        
+        await page.goto('https://meetup.hackrobot.cn/salon', wait_until='networkidle', timeout=30000)
+        
+        # 等待页面加载完成
+        await page.wait_for_timeout(3000)
+        
+        # 获取页面完整HTML
+        content = await page.content()
+        
+        # 保存HTML
+        with open('salon_full.html', 'w', encoding='utf-8') as f:
+            f.write(content)
+        
+        # 获取页面标题
+        title = await page.title()
+        print(f"页面标题: {title}")
+        
+        # 获取页面的视口高度
+        viewport_height = await page.evaluate('window.innerHeight')
+        page_height = await page.evaluate('document.body.scrollHeight')
+        
+        print(f"视口高度: {viewport_height}, 页面总高度: {page_height}")
+        
+        # 滚动并截图
+        screenshots = []
+        scroll_position = 0
+        screenshot_count = 0
+        
+        while scroll_position < page_height:
+            await page.screenshot(path=f'screenshot_{screenshot_count}.png', full_page=False)
+            screenshots.append(f'screenshot_{screenshot_count}.png')
+            screenshot_count += 1
+            
+            scroll_position += viewport_height
+            await page.evaluate(f'window.scrollTo(0, {scroll_position})')
+            await page.wait_for_timeout(1000)
+        
+        # 最后截取完整页面
+        await page.screenshot(path='screenshot_full.png', full_page=True)
+        
+        print(f"截图完成: 共 {screenshot_count} 张分段截图 + 1张完整截图")
+        
+        # 获取所有文本内容
+        text_content = await page.evaluate('''() => {
+            return document.body.innerText;
+        }''')
+        
+        with open('page_text.txt', 'w', encoding='utf-8') as f:
+            f.write(text_content)
+        
+        print("文本内容已保存")
+        
+        await browser.close()
+
+if __name__ == '__main__':
+    asyncio.run(scrape_page())