IyMg6aG555uu5pqC5pe25pCB572uCgpUT0RP77ya6Kej5Yaz54is6Jmr6L+H56iL5Lit55qE5Lq65py65qCh6aqM6Zeu6aKY

README.md

aW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9zCmltcG9ydCB0aW1lCgpmcm9tIGJzNCBpbXBvcnQgQmVhdXRpZnVsU291cApmcm9tIHNlbGVuaXVtIGltcG9ydCB3ZWJkcml2ZXIKCiMg6YWN572u5pel5b+XCmxvZ2dpbmcuYmFzaWNDb25maWcoCiAgICBmaWxlbmFtZT0nY3Jhd2xlci5sb2cnLAogICAgbGV2ZWw9bG9nZ2luZy5JTkZPLAogICAgZm9ybWF0PSclKGFzY3RpbWUpcyAtICUobGV2ZWxuYW1lKXMgLSAlKG1lc3NhZ2UpcycKKQoKCmNsYXNzIFdpa2lDcmF3bGVyOgogICAgZGVmIF9faW5pdF9fKHNlbGYpOgogICAgICAgICMg5Yid5aeL5YyW5rWP6KeI5Zmo6amx5YqoCiAgICAgICAgb3B0aW9ucyA9IHdlYmRyaXZlci5DaHJvbWVPcHRpb25zKCkKICAgICAgICBvcHRpb25zLmFkZF9hcmd1bWVudCgnLS1oZWFkbGVzcycpICAjIOaXoOWktOaooeW8jwogICAgICAgIG9wdGlvbnMuYWRkX2FyZ3VtZW50KCctLW5vLXNhbmRib3gnKQogICAgICAgIG9wdGlvbnMuYWRkX2FyZ3VtZW50KCctLWRpc2FibGUtZGV2LXNobS11c2FnZScpCgogICAgICAgIHNlbGYuZHJpdmVyID0gd2ViZHJpdmVyLkNocm9tZShvcHRpb25zPW9wdGlvbnMpCiAgICAgICAgc2VsZi5iYXNlX3VybCA9ICJodHRwczovL21pbGt5d2F5aWRsZS53aWtpLmdnLyIKCiAgICAgICAgIyDliJvlu7rovpPlh7rnm67lvZUKICAgICAgICBpZiBub3Qgb3MucGF0aC5leGlzdHMoJ291dHB1dCcpOgogICAgICAgICAgICBvcy5tYWtlZGlycygnb3V0cHV0JykKCiAgICBkZWYgY3Jhd2xfcGFnZShzZWxmLCBwYWdlX25hbWUpOgogICAgICAgICIiIueIrOWPluaMh+Wumndpa2npobXpnaIiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIHByaW50KGYi5q2j5Zyo54is5Y+W6aG16Z2iOiB7cGFnZV9uYW1lfSIpCiAgICAgICAgICAgIHVybCA9IGYie3NlbGYuYmFzZV91cmx9e3BhZ2VfbmFtZX0iCiAgICAgICAgICAgIHNlbGYuZHJpdmVyLmdldCh1cmwpCiAgICAgICAgICAgIHRpbWUuc2xlZXAoMikgICMg562J5b6F6aG16Z2i5Yqg6L29CgogICAgICAgICAgICAjIOiOt+WPlumhtemdokhUTUwKICAgICAgICAgICAgc291cCA9IEJlYXV0aWZ1bFNvdXAoc2VsZi5kcml2ZXIucGFnZV9zb3VyY2UsICdodG1sLnBhcnNlcicpCgogICAgICAgICAgICAjIOWkhOeQhnBhZ2VfbmFtZeS4reeahOWujOaVtFVSTAogICAgICAgICAgICBpZiBwYWdlX25hbWUuc3RhcnRzd2l0aCgiaHR0cHM6Ly8iKToKICAgICAgICAgICAgICAgIHBhZ2VfbmFtZSA9IHBhZ2VfbmFtZS5yZXBsYWNlKCJodHRwczovL21pbGt5d2F5aWRsZS53aWtpLmdnL3dpa2kiLCAiIikKICAgICAgICAgICAgICAgIAogICAgICAgICAgICAjIOS/neWtmOmhtemdouWGheWuuQogICAgICAgICAgICB3aXRoIG9wZW4oZiJvdXRwdXQve3BhZ2VfbmFtZX0uaHRtbCIsICJ3IiwgZW5jb2Rpbmc9InV0Zi04IikgYXMgZjoKICAgICAgICAgICAgICAgIGYud3JpdGUoc3RyKHNvdXApKQogICAgICAgICAgICAKICAgICAgICAgICAgIyDmj5Dlj5blubbkv53lrZhVUkxzCiAgICAgICAgICAgIHVybHMgPSBzZWxmLmV4dHJhY3RfdXJscyhzb3VwKQogICAgICAgICAgICB3aXRoIG9wZW4oZiJvdXRwdXQve3BhZ2VfbmFtZX1fdXJscy50eHQiLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAgICAgICAgICBmLndyaXRlKCdcbicuam9pbihbZiJodHRwczovL21pbGt5d2F5aWRsZS53aWtpLmdnL3dpa2l7dXJsfSIgZm9yIHVybCBpbiB1cmxzXSkpCgogICAgICAgICAgICBwcmludChmIuaIkOWKn+eIrOWPlumhtemdojoge3BhZ2VfbmFtZX0iKQogICAgICAgICAgICBwcmludChmIuaPkOWPluWIsCB7bGVuKHVybHMpfSDkuKpVUkwiKQogICAgICAgICAgICBsb2dnaW5nLmluZm8oZiLmiJDlip/niKzlj5bpobXpnaI6IHtwYWdlX25hbWV9IikKICAgICAgICAgICAgbG9nZ2luZy5pbmZvKGYi5o+Q5Y+W5YiwIHtsZW4odXJscyl9IOS4qlVSTCIpCiAgICAgICAgICAgIHJldHVybiBUcnVlCgogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgcHJpbnQoZiLniKzlj5bpobXpnaIge3BhZ2VfbmFtZX0g5aSx6LSlOiB7c3RyKGUpfSIpCiAgICAgICAgICAgIGxvZ2dpbmcuZXJyb3IoZiLniKzlj5bpobXpnaIge3BhZ2VfbmFtZX0g5aSx6LSlOiB7c3RyKGUpfSIpCiAgICAgICAgICAgIHJldHVybiBGYWxzZQoKICAgIGRlZiBleHRyYWN0X3VybHMoc2VsZiwgc291cCk6CiAgICAgICAgIiIi5o+Q5Y+W6aG16Z2i5Lit55qE5omA5pyJ5YaF6IGUVVJMIiIiCiAgICAgICAgdXJscyA9IFtdCiAgICAgICAgZm9yIGxpbmsgaW4gc291cC5maW5kX2FsbCgnYScpOgogICAgICAgICAgICBocmVmID0gbGluay5nZXQoJ2hyZWYnKQogICAgICAgICAgICAjIOWPquS/neeVmeKAnC93aWtp4oCd5byA5aS055qEVVJMCiAgICAgICAgICAgIGlmIGhyZWYgYW5kIGhyZWYuc3RhcnRzd2l0aCgiL3dpa2kiKToKICAgICAgICAgICAgICAgICMg6L2s5o2i5Li655u45a+56Lev5b6ECiAgICAgICAgICAgICAgICBocmVmID0gaHJlZi5yZXBsYWNlKCIvd2lraSIsICIiKQogICAgICAgICAgICAgICAgdXJscy5hcHBlbmQoaHJlZikKICAgICAgICAgICAgICAgIAogICAgICAgICMg5Y676YeN5o6S5bqPCiAgICAgICAgdXJscyA9IGxpc3Qoc2V0KHVybHMpKQogICAgICAgIHVybHMuc29ydCgpCiAgICAgICAgcmV0dXJuIHVybHMKCiAgICBkZWYgY2xvc2Uoc2VsZik6CiAgICAgICAgIiIi5YWz6Zet5rWP6KeI5ZmoIiIiCiAgICAgICAgc2VsZi5kcml2ZXIucXVpdCgpCgoKZGVmIGxvYWRfdXJsc19mcm9tX2ZpbGUoZmlsZW5hbWUpOgogICAgIiIi5LuO5paH5Lu25Yqg6L29VVJM5YiX6KGoIiIiCiAgICBpZiBub3Qgb3MucGF0aC5leGlzdHMoZmlsZW5hbWUpOgogICAgICAgIHJldHVybiBbXQogICAgd2l0aCBvcGVuKGZpbGVuYW1lLCAncicsIGVuY29kaW5nPSd1dGYtOCcpIGFzIGY6CiAgICAgICAgcmV0dXJuIFtsaW5lLnN0cmlwKCkgZm9yIGxpbmUgaW4gZiBpZiBsaW5lLnN0cmlwKCldCgpkZWYgc2F2ZV91cmxzX3RvX2ZpbGUoZmlsZW5hbWUsIHVybHMpOgogICAgIiIi5L+d5a2YVVJM5YiX6KGo5Yiw5paH5Lu2IiIiCiAgICB3aXRoIG9wZW4oZmlsZW5hbWUsICd3JywgZW5jb2Rpbmc9J3V0Zi04JykgYXMgZjoKICAgICAgICBmLndyaXRlKCdcbicuam9pbih1cmxzKSkKCmRlZiBhcHBlbmRfdXJsc190b19maWxlKGZpbGVuYW1lLCB1cmxzKToKICAgICIiIui/veWKoFVSTOWIl+ihqOWIsOaWh+S7tiIiIgogICAgZXhpc3RpbmcgPSBzZXQobG9hZF91cmxzX2Zyb21fZmlsZShmaWxlbmFtZSkpCiAgICBuZXdfdXJscyA9IFt1cmwgZm9yIHVybCBpbiB1cmxzIGlmIHVybCBub3QgaW4gZXhpc3RpbmddCiAgICBpZiBuZXdfdXJsczoKICAgICAgICB3aXRoIG9wZW4oZmlsZW5hbWUsICdhJywgZW5jb2Rpbmc9J3V0Zi04JykgYXMgZjoKICAgICAgICAgICAgZi53cml0ZSgnXG4nICsgJ1xuJy5qb2luKG5ld191cmxzKSkKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBjcmF3bGVyID0gV2lraUNyYXdsZXIoKQogICAgdHJ5OgogICAgICAgICMg5Yqg6L295b6F54is5Y+W5ZKM5bey54is5Y+WVVJM5YiX6KGoCiAgICAgICAgcXVldWUgPSBsb2FkX3VybHNfZnJvbV9maWxlKCdxdWV1ZS50eHQnKQogICAgICAgIHZpc2l0ZWQgPSBsb2FkX3VybHNfZnJvbV9maWxlKCd2aXNpdGVkLnR4dCcpCiAgICAgICAgCiAgICAgICAgIyDlpoLmnpzmsqHmnInlvoXniKzlj5ZVUkzvvIzku47pppbpobXlvIDlp4sKICAgICAgICBpZiBub3QgcXVldWU6CiAgICAgICAgICAgIHF1ZXVlID0gWyJNYWluX1BhZ2UiXQogICAgICAgICAgICAKICAgICAgICB3aGlsZSBxdWV1ZToKICAgICAgICAgICAgcGFnZV9uYW1lID0gcXVldWUucG9wKDApCiAgICAgICAgICAgIAogICAgICAgICAgICAjIOi3s+i/h+W3sueIrOWPlueahFVSTAogICAgICAgICAgICBpZiBwYWdlX25hbWUgaW4gdmlzaXRlZDoKICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgICAgICAKICAgICAgICAgICAgIyDniKzlj5bpobXpnaIKICAgICAgICAgICAgaWYgY3Jhd2xlci5jcmF3bF9wYWdlKHBhZ2VfbmFtZSk6CiAgICAgICAgICAgICAgICAjIOa3u+WKoOWIsOW3sueIrOWPluWIl+ihqAogICAgICAgICAgICAgICAgdmlzaXRlZC5hcHBlbmQocGFnZV9uYW1lKQogICAgICAgICAgICAgICAgc2F2ZV91cmxzX3RvX2ZpbGUoJ3Zpc2l0ZWQudHh0JywgdmlzaXRlZCkKICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgIyDmj5Dlj5bmlrBVUkzlubbmt7vliqDliLDpmJ/liJcKICAgICAgICAgICAgICAgIHVybHNfZmlsZSA9IGYib3V0cHV0L3twYWdlX25hbWV9X3VybHMudHh0IgogICAgICAgICAgICAgICAgaWYgb3MucGF0aC5leGlzdHModXJsc19maWxlKToKICAgICAgICAgICAgICAgICAgICBuZXdfdXJscyA9IGxvYWRfdXJsc19mcm9tX2ZpbGUodXJsc19maWxlKQogICAgICAgICAgICAgICAgICAgIGFwcGVuZF91cmxzX3RvX2ZpbGUoJ3F1ZXVlLnR4dCcsIG5ld191cmxzKQogICAgICAgICAgICAgICAgICAgIHF1ZXVlLmV4dGVuZChuZXdfdXJscykKICAgICAgICAgICAgICAgICAgICAKICAgIGZpbmFsbHk6CiAgICAgICAgY3Jhd2xlci5jbG9zZSgpCg==

crawler.py

Milky Way Idle 游戏维基 内容批量爬取并转换为 Markdown 格式，支持站内链接递归抓取，构建离线知识库。