918 lines
32 KiB
Lua
918 lines
32 KiB
Lua
local urlparse = require("socket.url")
|
|
local http = require("socket.http")
|
|
local cjson = require("cjson")
|
|
local utf8 = require("utf8")
|
|
|
|
local item_dir = os.getenv("item_dir")
|
|
local warc_file_base = os.getenv("warc_file_base")
|
|
local concurrency = tonumber(os.getenv("concurrency"))
|
|
local item_type = nil
|
|
local item_name = nil
|
|
local item_value = nil
|
|
local ids = {}
|
|
|
|
local url_count = 0
|
|
local tries = 0
|
|
local downloaded = {}
|
|
local seen_200 = {}
|
|
local addedtolist = {}
|
|
local abortgrab = false
|
|
local killgrab = false
|
|
local logged_response = false
|
|
|
|
local discovered_items = {}
|
|
local bad_items = {}
|
|
|
|
local retry_url = false
|
|
|
|
local item_patterns = {
|
|
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/viewer%.kt%?.*timesId=([0-9]+)"]="times",
|
|
["^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?.*timesId=([0-9]+)"]="times",
|
|
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/detail%.kt%?.*novelId=([0-9]+)"]="novel",
|
|
["^https?://[^/]*blice%.co%.kr/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event",
|
|
["^https?://[^/]*blice%.co%.kr/mw/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event",
|
|
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/documents/detail%.kt%?.*documentSeq=([0-9]+)"]="document",
|
|
["^https?://[^/]*blice%.co%.kr/web/support/noticeDetail%.kt%?.*noticeSeq=([0-9]+)"]="notice",
|
|
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/retrieveSpecificUserProfile%.kt%?.*userNicknameSeq=([0-9]+)"]="user",
|
|
["^https?://(cds%.blice%.co%.kr/download%?file=.+)"]="asset"
|
|
}
|
|
|
|
abort_item = function(item)
|
|
abortgrab = true
|
|
if not item then
|
|
item = item_name
|
|
end
|
|
if not bad_items[item] then
|
|
io.stdout:write("Aborting item " .. item .. ".\n")
|
|
io.stdout:flush()
|
|
bad_items[item] = true
|
|
end
|
|
end
|
|
|
|
kill_grab = function(item)
|
|
io.stdout:write("Aborting crawling.\n")
|
|
io.stdout:flush()
|
|
killgrab = true
|
|
end
|
|
|
|
read_file = function(file)
|
|
local f = assert(io.open(file, "rb"))
|
|
local data = f:read("*all")
|
|
f:close()
|
|
return data
|
|
end
|
|
|
|
processed = function(url)
|
|
if downloaded[url] or addedtolist[url] then
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
discover_item = function(target, item)
|
|
if item ~= item_name and not target[item] then
|
|
--print("discovered", item)
|
|
target[item] = true
|
|
return true
|
|
end
|
|
return false
|
|
end
|
|
|
|
query_param = function(url, name)
|
|
local query = string.match(url, "%?(.+)$")
|
|
if not query then
|
|
return nil
|
|
end
|
|
for key, value in string.gmatch(query, "([^&=]+)=?([^&]*)") do
|
|
if key == name then
|
|
return urlparse.unescape(value)
|
|
end
|
|
end
|
|
return nil
|
|
end
|
|
|
|
find_item = function(url)
|
|
url = urlparse.unescape(url)
|
|
for pattern, type_ in pairs(item_patterns) do
|
|
local value = string.match(url, pattern)
|
|
if value then
|
|
return {
|
|
["type"]=type_,
|
|
["value"]=value
|
|
}
|
|
end
|
|
end
|
|
end
|
|
|
|
set_item = function(url)
|
|
local found = find_item(url)
|
|
if found then
|
|
local new_item_type = found["type"]
|
|
local new_item_value = found["value"]
|
|
local new_item_name = new_item_type .. ":" .. new_item_value
|
|
if new_item_name ~= item_name then
|
|
item_value = new_item_value
|
|
item_type = new_item_type
|
|
ids = {}
|
|
ids[item_value] = true
|
|
abortgrab = false
|
|
tries = 0
|
|
retry_url = false
|
|
item_name = new_item_name
|
|
print("Archiving item " .. item_name)
|
|
end
|
|
end
|
|
end
|
|
|
|
allowed = function(url, body_data)
|
|
if not string.match(url, "^https?://") then
|
|
return false
|
|
end
|
|
|
|
if not (
|
|
string.match(url, "^https?://www%.blice%.co%.kr/")
|
|
or string.match(url, "^https?://blice%.co%.kr/")
|
|
or string.match(url, "^https?://cds%.blice%.co%.kr/")
|
|
) then
|
|
return false
|
|
end
|
|
|
|
local found = find_item(url)
|
|
if found then
|
|
local found_item_name = found["type"] .. ":" .. found["value"]
|
|
if found_item_name ~= item_name then
|
|
discover_item(discovered_items, found_item_name)
|
|
return false
|
|
end
|
|
return true
|
|
end
|
|
|
|
for _, pattern in ipairs({
|
|
"^https?://[^/]*blice%.co%.kr/[mw][we]?b?/comment%.kt%?",
|
|
"^https?://[^/]*blice%.co%.kr/[mw][we]?b?/other_more_list%.kt%?",
|
|
"^https?://[^/]*blice%.co%.kr/web/readMoreTimesList_ajax%?",
|
|
"^https?://[^/]*blice%.co%.kr/mw/readMoreTimesList_ajax%?",
|
|
"^https?://[^/]*blice%.co%.kr/web/viewer_readMoreTimesList_ajax%?",
|
|
"^https?://[^/]*blice%.co%.kr/web/commentList_ajax%?",
|
|
"^https?://[^/]*blice%.co%.kr/web/event/commentList_ajax%?",
|
|
"^https?://[^/]*blice%.co%.kr/[mw][we]?b?/commentListDetail%.kt%?"
|
|
}) do
|
|
if string.match(url, pattern) then
|
|
for _, name in ipairs({
|
|
"novelId",
|
|
"novel_no",
|
|
"novelSeq",
|
|
"timesId",
|
|
"novelTimesSeq",
|
|
"exceptionalEventSeq",
|
|
"eventSeq"
|
|
}) do
|
|
if ids[query_param(url, name)] then
|
|
return true
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
if body_data then
|
|
return true
|
|
end
|
|
|
|
if item_type == "event"
|
|
and (
|
|
string.match(url, "^https?://[^/]*blice%.co%.kr/event/more%.kt%?")
|
|
or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?")
|
|
) then
|
|
return true
|
|
end
|
|
|
|
return false
|
|
end
|
|
|
|
wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
|
|
return false
|
|
end
|
|
|
|
decode_codepoint = function(newurl)
|
|
newurl = string.gsub(
|
|
newurl, "\\[uU]([0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F])",
|
|
function (s)
|
|
return utf8.char(tonumber(s, 16))
|
|
end
|
|
)
|
|
return newurl
|
|
end
|
|
|
|
wget.callbacks.get_urls = function(file, url, is_css, iri)
|
|
local urls = {}
|
|
|
|
downloaded[url] = true
|
|
set_item(url)
|
|
|
|
if abortgrab then
|
|
return {}
|
|
end
|
|
|
|
local function fix_case(newurl)
|
|
if not string.match(newurl, "^https?://[^/]") then
|
|
return newurl
|
|
end
|
|
if string.match(newurl, "^https?://[^/]+$") then
|
|
newurl = newurl .. "/"
|
|
end
|
|
local a, b = string.match(newurl, "^(https?://[^/]+/)(.*)$")
|
|
return string.lower(a) .. b
|
|
end
|
|
|
|
local function check(newurl, headers, body_data)
|
|
if not string.match(newurl, "^https?://") then
|
|
return nil
|
|
end
|
|
newurl = decode_codepoint(newurl)
|
|
if string.match(newurl, "[%s\\]") then
|
|
return nil
|
|
end
|
|
newurl = fix_case(newurl)
|
|
if not body_data
|
|
and string.match(newurl, "^https?://www%.blice%.co%.kr/")
|
|
and not string.match(newurl, "_ajax%?")
|
|
and not string.match(newurl, "/api/") then
|
|
newurl = string.gsub(newurl, "^https?://www%.blice%.co%.kr/", "https://blice.co.kr/")
|
|
end
|
|
local url = string.match(newurl, "^([^#]+)")
|
|
local url_ = url
|
|
while string.match(url_, "&") do
|
|
url_ = string.gsub(url_, "&", "&")
|
|
end
|
|
local method = body_data and "POST" or "GET"
|
|
local key = method .. "\0" .. url_ .. "\0" .. tostring(body_data)
|
|
if not processed(key)
|
|
and (body_data or not processed(url_))
|
|
and allowed(url_, body_data) then
|
|
local url_data = {
|
|
url=url_,
|
|
headers=headers or {}
|
|
}
|
|
if body_data then
|
|
url_data["body_data"] = body_data
|
|
url_data["method"] = method
|
|
end
|
|
table.insert(urls, url_data)
|
|
addedtolist[key] = true
|
|
if not body_data then
|
|
addedtolist[url_] = true
|
|
addedtolist[url] = true
|
|
end
|
|
return true
|
|
end
|
|
end
|
|
|
|
local function checknewurl(newurl)
|
|
newurl = decode_codepoint(newurl)
|
|
if string.match(newurl, "['\"><]") then
|
|
return nil
|
|
end
|
|
if string.match(newurl, "^https?:////") then
|
|
check(string.gsub(newurl, ":////", "://"))
|
|
elseif string.match(newurl, "^https?://") then
|
|
check(newurl)
|
|
elseif string.match(newurl, "^https?:\\/\\?/") then
|
|
check(string.gsub(newurl, "\\", ""))
|
|
elseif string.match(newurl, "^\\/\\/") then
|
|
checknewurl(string.gsub(newurl, "\\", ""))
|
|
elseif string.match(newurl, "^//") then
|
|
check(urlparse.absolute(url, newurl))
|
|
elseif string.match(newurl, "^\\/") then
|
|
checknewurl(string.gsub(newurl, "\\", ""))
|
|
elseif string.match(newurl, "^/") then
|
|
check(urlparse.absolute(url, newurl))
|
|
elseif string.match(newurl, "^%.%./") then
|
|
if string.match(url, "^https?://[^/]+/[^/]+/") then
|
|
check(urlparse.absolute(url, newurl))
|
|
else
|
|
checknewurl(string.match(newurl, "^%.%.(/.+)$"))
|
|
end
|
|
elseif string.match(newurl, "^%./") then
|
|
check(urlparse.absolute(url, newurl))
|
|
end
|
|
end
|
|
|
|
local function checknewshorturl(newurl)
|
|
newurl = decode_codepoint(newurl)
|
|
if string.match(newurl, "^%?") then
|
|
check(urlparse.absolute(url, newurl))
|
|
elseif not (
|
|
string.match(newurl, "^https?:\\?/\\?//?/?")
|
|
or string.match(newurl, "^[/\\]")
|
|
or string.match(newurl, "^%./")
|
|
or string.match(newurl, "^[jJ]ava[sS]cript:")
|
|
or string.match(newurl, "^[mM]ail[tT]o:")
|
|
or string.match(newurl, "^data:")
|
|
or string.match(newurl, "^%${")
|
|
) then
|
|
check(urlparse.absolute(url, newurl))
|
|
end
|
|
end
|
|
|
|
local function check_media(data)
|
|
local values = {data}
|
|
if type(data) == "table" then
|
|
for _, key in ipairs({
|
|
"epubCoverUrl",
|
|
"epubUrl",
|
|
"filePath1",
|
|
"filePath2",
|
|
"imagePath1",
|
|
"imagePath2",
|
|
"profileImgPath",
|
|
"sticonUrl",
|
|
"timesCoverImagePath",
|
|
"timesFilePath",
|
|
"userImage",
|
|
"workImageWideUrl",
|
|
"workThumbUrl"
|
|
}) do
|
|
if data[key] then
|
|
table.insert(values, data[key])
|
|
end
|
|
end
|
|
if type(data["webtoonImages"]) == "table" then
|
|
for _, value in ipairs(data["webtoonImages"]) do
|
|
table.insert(values, value)
|
|
end
|
|
end
|
|
for _, pair in ipairs({
|
|
{"epubSavePath", "epubNewName"},
|
|
{"epubCoverSavePath", "epubCoverNewName"}
|
|
}) do
|
|
local save_path = data[pair[1]]
|
|
local new_name = data[pair[2]]
|
|
if type(save_path) == "string" and type(new_name) == "string" then
|
|
save_path = string.gsub(save_path, "^/nas/noveltimes/epub/", "/epub/noveltimes/")
|
|
if string.match(save_path, "^/epub/noveltimes/") then
|
|
table.insert(values, save_path .. new_name)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
for _, value in ipairs(values) do
|
|
if type(value) == "string" then
|
|
if string.match(value, "^https?://") then
|
|
check(value)
|
|
elseif string.match(value, "^/") then
|
|
check("https://cds.blice.co.kr/download?file=" .. value)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
local function check_xmlhttprequest(url, referer, body)
|
|
local headers = {
|
|
["Content-Type"]="application/json; charset=UTF-8",
|
|
["X-Requested-With"]="XMLHttpRequest",
|
|
["Referer"]=referer
|
|
}
|
|
check(string.match(url, "^([^?]+)"), headers, body)
|
|
check(url, headers, body)
|
|
end
|
|
|
|
local function check_web_mw(path_query, web_extra, mw_extra)
|
|
web_extra = web_extra or ""
|
|
if not mw_extra then
|
|
mw_extra = web_extra
|
|
end
|
|
for _, data in ipairs({
|
|
{"web", web_extra},
|
|
{"mw", mw_extra}
|
|
}) do
|
|
check("https://blice.co.kr/" .. data[1] .. "/" .. path_query .. data[2])
|
|
end
|
|
end
|
|
|
|
local function check_readmore(endpoint, page_no, order_by, novel_type_seq)
|
|
local path = "readMoreTimesList_ajax"
|
|
local body_novel_no = item_value
|
|
if endpoint == "viewer" then
|
|
endpoint = "web"
|
|
path = "viewer_readMoreTimesList_ajax"
|
|
order_by = "recent"
|
|
body_novel_no = '"' .. item_value .. '"'
|
|
end
|
|
local url = "https://blice.co.kr/" .. endpoint .. "/" .. path
|
|
.. "?novel_no=" .. item_value
|
|
.. "&orderBy=" .. order_by
|
|
.. "&pageNo=" .. tostring(page_no)
|
|
local body = '{"novel_no":' .. body_novel_no
|
|
if novel_type_seq then
|
|
url = url .. "&novelTypeSeq=" .. novel_type_seq
|
|
body = body .. ',"novelTypeSeq":' .. novel_type_seq
|
|
end
|
|
body = body
|
|
.. ',"orderBy":"' .. order_by .. '"'
|
|
.. ',"pageNo":' .. tostring(page_no)
|
|
if endpoint == "mw" or path == "viewer_readMoreTimesList_ajax" then
|
|
body = body .. ',"row":20'
|
|
url = url .. "&row=20"
|
|
end
|
|
body = body .. "}"
|
|
check_xmlhttprequest(
|
|
url,
|
|
"https://blice.co.kr/" .. endpoint .. "/detail.kt?novelId=" .. item_value,
|
|
body
|
|
)
|
|
end
|
|
|
|
local function check_comment_ajax(novel_seq, times_seq, comment_type, page_no)
|
|
local referer = "https://blice.co.kr/web/comment.kt?novelId=" .. novel_seq
|
|
.. "&commentType=" .. comment_type
|
|
if times_seq ~= "0" then
|
|
referer = "https://blice.co.kr/web/comment.kt?timesId=" .. times_seq
|
|
.. "&novelId=" .. novel_seq
|
|
.. "&commentType=" .. comment_type
|
|
end
|
|
local body = '{"novelSeq":' .. novel_seq
|
|
.. ',"novelTimesSeq":' .. times_seq
|
|
.. ',"commentType":"' .. comment_type .. '"'
|
|
.. ',"orderBy":"recent"'
|
|
.. ',"pageNo":' .. tostring(page_no)
|
|
.. "}"
|
|
check_xmlhttprequest(
|
|
"https://blice.co.kr/web/commentList_ajax?novelSeq=" .. novel_seq
|
|
.. "&novelTimesSeq=" .. times_seq
|
|
.. "&commentType=" .. comment_type
|
|
.. "&orderBy=recent"
|
|
.. "&pageNo=" .. tostring(page_no),
|
|
referer,
|
|
body
|
|
)
|
|
end
|
|
|
|
if status_code == 200
|
|
and allowed(url)
|
|
and (
|
|
string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+%.kt")
|
|
or string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+_ajax%?")
|
|
or string.match(url, "^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?")
|
|
or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?")
|
|
) then
|
|
local html = read_file(file)
|
|
if string.match(url, "^https?://blice%.co%.kr/web/detail%.kt%?")
|
|
and query_param(url, "novelId") == item_value then
|
|
check_web_mw(
|
|
"detail.kt?novelId=" .. item_value,
|
|
"&isPrologTimes=false&voucherXCnt=0&voucherYCnt=0&voucherY1Cnt=0&voucherY3Cnt=0&voucherLCnt=0&voucherSCnt=0"
|
|
.. "&viewVoucherRentCnt=0&viewVoucherHaveCnt=0&needGaLog=true&isLogin=false"
|
|
.. "&prevUrl=http%3A%2F%2Fblice.co.kr%2Fweb%2Fviewer.kt",
|
|
""
|
|
)
|
|
check_web_mw("comment.kt?novelId=" .. item_value .. "&commentType=novel", "", "&quickViewYn=N")
|
|
for _, data in ipairs({
|
|
{"web", "seq"},
|
|
{"web", "recent"},
|
|
{"web", "regDate"},
|
|
{"mw", "seq"},
|
|
{"mw", "recent"}
|
|
}) do
|
|
check_readmore(data[1], 1, data[2])
|
|
end
|
|
local novel_type_seq = string.match(html, "novelTypeSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)")
|
|
or string.match(html, "novelTypeSeq=([0-9]+)")
|
|
if novel_type_seq then
|
|
for _, order_by in ipairs({"recent", "regDate"}) do
|
|
check_readmore("web", 1, order_by, novel_type_seq)
|
|
end
|
|
end
|
|
check_readmore("viewer", 1)
|
|
check_comment_ajax(item_value, "0", "novel", 1)
|
|
elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?")
|
|
and query_param(url, "timesId") == item_value then
|
|
local times_type = query_param(url, "timesType")
|
|
if times_type == "" then
|
|
times_type = nil
|
|
end
|
|
times_type = times_type
|
|
or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)")
|
|
or string.match(html, "timesType=([a-zA-Z]+)")
|
|
if not times_type
|
|
or not string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)") then
|
|
return urls
|
|
end
|
|
local viewer_path_query = "viewer.kt?timesId=" .. item_value .. "×Type=" .. times_type
|
|
check_web_mw(viewer_path_query, "&quickViewYn=N")
|
|
check_web_mw(viewer_path_query, "&quickViewYn=")
|
|
check_web_mw(viewer_path_query)
|
|
check("https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value)
|
|
check_xmlhttprequest(
|
|
"https://blice.co.kr/api/novel/v1/viewerAjax?timesId=" .. item_value .. "×Type=" .. times_type .. "&quickViewYn=N",
|
|
"https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value .. "×Type=" .. times_type .. "&quickViewYn=N",
|
|
'{"timesId":' .. item_value .. ',"timesType":"' .. times_type .. '","quickViewYn":"N"}'
|
|
)
|
|
elseif string.match(url, "^https?://blice%.co%.kr/event/eventPage%.kt%?")
|
|
and query_param(url, "eventSeq") == item_value then
|
|
check("https://blice.co.kr/mw/event/eventPage.kt?eventSeq=" .. item_value)
|
|
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1")
|
|
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1&exceptionalEventSeq=" .. item_value .. "&pageNo=1")
|
|
elseif string.match(url, "^https?://blice%.co%.kr/web/documents/detail%.kt%?")
|
|
and query_param(url, "documentSeq") == item_value then
|
|
check_web_mw("documents/detail.kt?documentSeq=" .. item_value, "&start=1", "")
|
|
elseif string.match(url, "^https?://blice%.co%.kr/mw/retrieveSpecificUserProfile%.kt%?")
|
|
and query_param(url, "userNicknameSeq") == item_value then
|
|
if query_param(url, "userType") == "AUTHOR_PEN" then
|
|
local penname_id = query_param(url, "userPennameSeq")
|
|
check_xmlhttprequest(
|
|
"https://blice.co.kr/web/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userPennameSeq=" .. penname_id .. "&userNicknameSeq=" .. item_value,
|
|
"https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. penname_id,
|
|
'{"userType":"AUTHOR_PEN","userPennameSeq":"' .. penname_id .. '","userNicknameSeq":"' .. item_value .. '","adultViewYn":1,"onlyPenYn":"Y"}'
|
|
)
|
|
else
|
|
for _, user_type in ipairs({
|
|
"USER",
|
|
"AUTHOR_NICK"
|
|
}) do
|
|
check_xmlhttprequest(
|
|
"https://blice.co.kr/web/retrieveSpecificUserProfile.kt?userType=" .. user_type .. "&userNicknameSeq=" .. item_value,
|
|
"https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item_value,
|
|
'{"userType":"' .. user_type .. '","userNicknameSeq":"' .. item_value .. '"}'
|
|
)
|
|
end
|
|
end
|
|
end
|
|
if string.match(url, "/readMoreTimesList_ajax%?")
|
|
or string.match(url, "/viewer_readMoreTimesList_ajax%?") then
|
|
local response = cjson.decode(html)["response"]
|
|
local count = 0
|
|
local times_list = response["timesResList"]
|
|
if string.match(url, "/mw/")
|
|
or string.match(url, "/viewer_readMoreTimesList_ajax%?") then
|
|
times_list = response
|
|
end
|
|
for _, item in ipairs(times_list) do
|
|
count = count + 1
|
|
check("https://blice.co.kr/web/viewer.kt?timesId=" .. item["novelTimesSeq"])
|
|
check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
|
|
check_media(item)
|
|
end
|
|
local page_no = tonumber(query_param(url, "pageNo"))
|
|
local order_by = query_param(url, "orderBy")
|
|
local novel_type_seq = query_param(url, "novelTypeSeq")
|
|
local is_mw = string.match(url, "/mw/")
|
|
local is_viewer = string.match(url, "/viewer_readMoreTimesList_ajax%?")
|
|
local readmore_endpoint = "web"
|
|
if is_mw then
|
|
readmore_endpoint = "mw"
|
|
elseif is_viewer then
|
|
readmore_endpoint = "viewer"
|
|
end
|
|
if not is_mw and not is_viewer then
|
|
for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do
|
|
new_page = tonumber(new_page)
|
|
if new_page > page_no then
|
|
check_readmore("web", new_page, order_by, novel_type_seq)
|
|
end
|
|
end
|
|
end
|
|
if count > 0 then
|
|
if readmore_endpoint == "mw" then
|
|
check_readmore(readmore_endpoint, page_no + 1, order_by)
|
|
elseif readmore_endpoint == "viewer" then
|
|
check_readmore(readmore_endpoint, page_no + 1)
|
|
else
|
|
check_readmore(readmore_endpoint, page_no + 1, order_by, novel_type_seq)
|
|
end
|
|
end
|
|
elseif string.match(url, "/event/commentList_ajax%?") then
|
|
local response = cjson.decode(html)["response"]
|
|
local count = 0
|
|
for _, item in ipairs(response["eventCommentResList"]) do
|
|
count = count + 1
|
|
check_media(item)
|
|
end
|
|
local page_no = tonumber(query_param(url, "pageNo"))
|
|
local event_seq = query_param(url, "exceptionalEventSeq")
|
|
for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)") do
|
|
new_page = tonumber(new_page)
|
|
if new_page > page_no then
|
|
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(new_page))
|
|
end
|
|
end
|
|
if count >= 30 then
|
|
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(page_no + 1))
|
|
end
|
|
elseif string.match(url, "/event/card%?") then
|
|
local count = 0
|
|
for _, item in ipairs(cjson.decode(html)["response"]["workList"]) do
|
|
count = count + 1
|
|
check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
|
|
check_media(item)
|
|
end
|
|
if count > 0 then
|
|
check("https://blice.co.kr/event/card?cardListSeq=" .. query_param(url, "cardListSeq")
|
|
.. "&startNo=" .. tostring(tonumber(query_param(url, "startNo")) + 1))
|
|
end
|
|
elseif string.match(url, "/commentList_ajax%?") then
|
|
local response = cjson.decode(html)["response"]
|
|
local count = 0
|
|
local comment_type = query_param(url, "commentType")
|
|
local times_seq = query_param(url, "novelTimesSeq")
|
|
for _, item in ipairs(response["timesCommentResList"]) do
|
|
count = count + 1
|
|
check_media(item)
|
|
check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item["userNicknameSeq"])
|
|
check_web_mw(
|
|
"commentListDetail.kt?novelId=" .. item["novelSeq"] .. "×Id=" .. item["novelTimesSeq"],
|
|
"&commentId=" .. item["timesCommentSeq"] .. "&commentType=" .. comment_type,
|
|
"×Type=" .. comment_type
|
|
.. "&commentId=" .. item["timesCommentSeq"]
|
|
.. "&commentType=" .. comment_type
|
|
.. "&quickViewYn=N"
|
|
)
|
|
end
|
|
local page_no = tonumber(query_param(url, "pageNo"))
|
|
for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do
|
|
new_page = tonumber(new_page)
|
|
if new_page > page_no then
|
|
check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, new_page)
|
|
end
|
|
end
|
|
if count >= 30 then
|
|
check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, page_no + 1)
|
|
end
|
|
elseif string.match(url, "^https?://[^/]*blice%.co%.kr/web/retrieveSpecificUserProfile%.kt%?") then
|
|
local response = cjson.decode(html)
|
|
check_media(response["specificUserProfile"])
|
|
if query_param(url, "userType") ~= "USER" then
|
|
for _, item in ipairs(response["authorNovelList"]) do
|
|
check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y"
|
|
.. "&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. item["authorPenSeq"])
|
|
check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
|
|
check_media(item)
|
|
end
|
|
else
|
|
for _, item in ipairs(response["userDocumentsList"]) do
|
|
check("https://blice.co.kr/web/documents/detail.kt?documentSeq=" .. item["documentSeq"])
|
|
end
|
|
end
|
|
elseif string.match(url, "/api/novel/v1/viewerAjax%?") then
|
|
local response = cjson.decode(html)["response"]
|
|
check("https://blice.co.kr/web/detail.kt?novelId=" .. response["novelSeq"])
|
|
check_media(response)
|
|
for _, key in ipairs({"prev", "next"}) do
|
|
local adjacent = response[key]
|
|
if type(adjacent) == "table" then
|
|
check("https://blice.co.kr/web/viewer.kt?timesId=" .. adjacent["novelTimesSeq"])
|
|
end
|
|
end
|
|
else
|
|
for _, data in ipairs({
|
|
{"novelId=([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"},
|
|
{"data%-novelSeq=['\"]?([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"},
|
|
{"data%-first%-timesId=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"},
|
|
{"data%-times%-seq=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"},
|
|
{"data%-nickNameSeq=\"([0-9]+)\"", "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=%s"},
|
|
{"eventSeq=([0-9]+)", "https://blice.co.kr/event/eventPage.kt?eventSeq=%s"},
|
|
{"documentSeq=([0-9]+)", "https://blice.co.kr/web/documents/detail.kt?documentSeq=%s"},
|
|
{"noticeSeq=([0-9]+)", "https://blice.co.kr/web/support/noticeDetail.kt?noticeSeq=%s"}
|
|
}) do
|
|
for value in string.gmatch(html, data[1]) do
|
|
check(string.format(data[2], value))
|
|
end
|
|
end
|
|
for times_id in string.gmatch(html, "timesId=([0-9]+)") do
|
|
check("https://blice.co.kr/web/viewer.kt?timesId=" .. times_id)
|
|
end
|
|
if query_param(url, "novelId") == item_value
|
|
or query_param(url, "novel_no") == item_value
|
|
or query_param(url, "novelSeq") == item_value then
|
|
for penname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\"") do
|
|
check_web_mw("other_more_list.kt?novelId=" .. item_value .. "&pennameId=" .. penname_id)
|
|
end
|
|
for penname_id, nickname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\" data%-nickname%-seq=\"([0-9]+)\" data%-adult%-view%-yn=\"[^\"]*\"") do
|
|
check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y"
|
|
.. "&userType=AUTHOR_PEN&userNicknameSeq=" .. nickname_id .. "&userPennameSeq=" .. penname_id)
|
|
end
|
|
elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?") then
|
|
local novel_no = string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)")
|
|
if not novel_no then
|
|
return urls
|
|
end
|
|
check("https://blice.co.kr/web/detail.kt?novelId=" .. novel_no)
|
|
local times_type = query_param(url, "timesType")
|
|
if times_type == "" then
|
|
times_type = nil
|
|
end
|
|
times_type = times_type
|
|
or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)")
|
|
or string.match(html, "timesType=([a-zA-Z]+)")
|
|
if not times_type then
|
|
return urls
|
|
end
|
|
check_web_mw("comment.kt?timesId=" .. item_value .. "&novelId=" .. novel_no
|
|
.. "&commentType=" .. times_type, "", "&quickViewYn=N")
|
|
check("https://blice.co.kr/mw/comment.kt?timesId=" .. item_value
|
|
.. "&novelId=" .. novel_no
|
|
.. "&commentType=" .. times_type
|
|
.. "&quickViewYn=N&hback=%2Fmw%2Fviewer.kt%3FtimesId%3D" .. item_value
|
|
.. "%26timesType%3D" .. times_type
|
|
.. "%26quickViewYn%3DN")
|
|
check_comment_ajax(novel_no, item_value, times_type, 1)
|
|
elseif string.match(url, "^https?://blice%.co%.kr/event/")
|
|
or string.match(url, "^https?://blice%.co%.kr/mw/event/") then
|
|
for card_list_id in string.gmatch(html, "cardListSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)") do
|
|
check("https://blice.co.kr/event/card?cardListSeq=" .. card_list_id .. "&startNo=2")
|
|
end
|
|
end
|
|
for newurl in string.gmatch(string.gsub(html, "&[qQ][uU][oO][tT];", '"'), '([^"]+)') do
|
|
checknewurl(newurl)
|
|
end
|
|
for newurl in string.gmatch(string.gsub(html, "'", "'"), "([^']+)") do
|
|
checknewurl(newurl)
|
|
end
|
|
for newurl in string.gmatch(html, "[^%-]href='([^']+)'") do
|
|
checknewshorturl(newurl)
|
|
end
|
|
for newurl in string.gmatch(html, '[^%-]href="([^"]+)"') do
|
|
checknewshorturl(newurl)
|
|
end
|
|
for newurl in string.gmatch(html, "[^%-]src='([^']+)'") do
|
|
checknewshorturl(newurl)
|
|
end
|
|
for newurl in string.gmatch(html, '[^%-]src="([^"]+)"') do
|
|
checknewshorturl(newurl)
|
|
end
|
|
for newurl in string.gmatch(html, ":%s*url%(([^%)]+)%)") do
|
|
newurl = string.gsub(newurl, "^['\"]", "")
|
|
newurl = string.gsub(newurl, "['\"]$", "")
|
|
checknewurl(newurl)
|
|
end
|
|
html = string.gsub(html, ">", ">")
|
|
html = string.gsub(html, "<", "<")
|
|
for newurl in string.gmatch(html, ">%s*([^<%s]+)") do
|
|
checknewurl(newurl)
|
|
end
|
|
end
|
|
end
|
|
|
|
return urls
|
|
end
|
|
|
|
wget.callbacks.write_to_warc = function(url, http_stat)
|
|
status_code = http_stat["statcode"]
|
|
set_item(url["url"])
|
|
url_count = url_count + 1
|
|
io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n")
|
|
io.stdout:flush()
|
|
logged_response = true
|
|
if not item_name then
|
|
error("No item name found.")
|
|
end
|
|
if http_stat["len"] == 0
|
|
and status_code < 300 then
|
|
retry_url = true
|
|
return false
|
|
end
|
|
if status_code ~= 200
|
|
and status_code ~= 302 then
|
|
retry_url = true
|
|
return false
|
|
end
|
|
if abortgrab then
|
|
print("Not writing to WARC.")
|
|
return false
|
|
end
|
|
retry_url = false
|
|
tries = 0
|
|
return true
|
|
end
|
|
|
|
wget.callbacks.httploop_result = function(url, err, http_stat)
|
|
status_code = http_stat["statcode"]
|
|
|
|
if not logged_response then
|
|
url_count = url_count + 1
|
|
io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n")
|
|
io.stdout:flush()
|
|
end
|
|
logged_response = false
|
|
|
|
if killgrab then
|
|
return wget.actions.ABORT
|
|
end
|
|
|
|
set_item(url["url"])
|
|
if not item_name then
|
|
error("No item name found.")
|
|
end
|
|
|
|
if abortgrab then
|
|
abort_item()
|
|
return wget.actions.EXIT
|
|
end
|
|
|
|
if status_code == 0 or retry_url then
|
|
io.stdout:write("Server returned bad response. ")
|
|
io.stdout:flush()
|
|
tries = tries + 1
|
|
local maxtries = 6
|
|
if tries > maxtries then
|
|
io.stdout:write(" Skipping.\n")
|
|
io.stdout:flush()
|
|
tries = 0
|
|
abort_item()
|
|
return wget.actions.EXIT
|
|
end
|
|
local sleep_time = math.random(
|
|
math.floor(math.pow(2, tries-0.5)),
|
|
math.floor(math.pow(2, tries))
|
|
)
|
|
io.stdout:write("Sleeping " .. sleep_time .. " seconds.\n")
|
|
io.stdout:flush()
|
|
os.execute("sleep " .. sleep_time)
|
|
return wget.actions.CONTINUE
|
|
else
|
|
if status_code == 200 or status_code == 206 then
|
|
if not seen_200[url["url"]] then
|
|
seen_200[url["url"]] = 0
|
|
end
|
|
seen_200[url["url"]] = seen_200[url["url"]] + 1
|
|
end
|
|
downloaded[url["url"]] = true
|
|
end
|
|
|
|
if status_code >= 300 and status_code <= 399 then
|
|
local newloc = urlparse.absolute(url["url"], http_stat["newloc"])
|
|
if processed(newloc) or not allowed(newloc) then
|
|
tries = 0
|
|
return wget.actions.EXIT
|
|
end
|
|
end
|
|
|
|
tries = 0
|
|
|
|
return wget.actions.NOTHING
|
|
end
|
|
|
|
wget.callbacks.finish = function(start_time, end_time, wall_time, numurls, total_downloaded_bytes, total_download_time)
|
|
local function submit_backfeed(items, key)
|
|
local tries = 0
|
|
local maxtries = 5
|
|
while tries < maxtries do
|
|
if killgrab then
|
|
return false
|
|
end
|
|
local body, code, headers, status = http.request(
|
|
"https://legacy-api.arpa.li/backfeed/legacy/" .. key,
|
|
items .. "\0"
|
|
)
|
|
if code == 200 and body ~= nil and cjson.decode(body)["status_code"] == 200 then
|
|
io.stdout:write(string.match(body, "^(.-)%s*$") .. "\n")
|
|
io.stdout:flush()
|
|
return nil
|
|
end
|
|
io.stdout:write("Failed to submit discovered URLs." .. tostring(code) .. tostring(body) .. "\n")
|
|
io.stdout:flush()
|
|
os.execute("sleep " .. math.floor(math.pow(2, tries)))
|
|
tries = tries + 1
|
|
end
|
|
kill_grab()
|
|
error()
|
|
end
|
|
|
|
local file = io.open(item_dir .. "/" .. warc_file_base .. "_bad-items.txt", "w")
|
|
for url, _ in pairs(bad_items) do
|
|
file:write(url .. "\n")
|
|
end
|
|
file:close()
|
|
for key, data in pairs({
|
|
["blice-tob75dmr5nkp494q"] = discovered_items
|
|
}) do
|
|
print("queuing for", string.match(key, "^(.+)%-") or key)
|
|
local items = nil
|
|
local count = 0
|
|
for item, _ in pairs(data) do
|
|
print("found item", item)
|
|
if items == nil then
|
|
items = item
|
|
else
|
|
items = items .. "\0" .. item
|
|
end
|
|
count = count + 1
|
|
if count == 1000 then
|
|
submit_backfeed(items, key)
|
|
items = nil
|
|
count = 0
|
|
end
|
|
end
|
|
submit_backfeed(items, key)
|
|
end
|
|
end
|
|
|
|
wget.callbacks.before_exit = function(exit_status, exit_status_string)
|
|
if killgrab then
|
|
return wget.exits.IO_FAIL
|
|
end
|
|
if abortgrab then
|
|
abort_item()
|
|
end
|
|
return exit_status
|
|
end
|