mirrors/blice-grab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version 20260629.01. Initial.

Browse Source
This commit is contained in:
arkiver
2026-06-29 04:23:36 +02:00
commit 69b0038af5
6 changed files with 1377 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

917
blice.lua Normal file
View File

@@ -0,0 +1,917 @@
local urlparse = require("socket.url")
local http = require("socket.http")
local cjson = require("cjson")
local utf8 = require("utf8")
local item_dir = os.getenv("item_dir")
local warc_file_base = os.getenv("warc_file_base")
local concurrency = tonumber(os.getenv("concurrency"))
local item_type = nil
local item_name = nil
local item_value = nil
local ids = {}
local url_count = 0
local tries = 0
local downloaded = {}
local seen_200 = {}
local addedtolist = {}
local abortgrab = false
local killgrab = false
local logged_response = false
local discovered_items = {}
local bad_items = {}
local retry_url = false
local item_patterns = {
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/viewer%.kt%?.*timesId=([0-9]+)"]="times",
["^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?.*timesId=([0-9]+)"]="times",
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/detail%.kt%?.*novelId=([0-9]+)"]="novel",
["^https?://[^/]*blice%.co%.kr/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event",
["^https?://[^/]*blice%.co%.kr/mw/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event",
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/documents/detail%.kt%?.*documentSeq=([0-9]+)"]="document",
["^https?://[^/]*blice%.co%.kr/web/support/noticeDetail%.kt%?.*noticeSeq=([0-9]+)"]="notice",
["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/retrieveSpecificUserProfile%.kt%?.*userNicknameSeq=([0-9]+)"]="user",
["^https?://(cds%.blice%.co%.kr/download%?file=.+)"]="asset"
}
abort_item = function(item)
abortgrab = true
if not item then
item = item_name
end
if not bad_items[item] then
io.stdout:write("Aborting item " .. item .. ".\n")
io.stdout:flush()
bad_items[item] = true
end
end
kill_grab = function(item)
io.stdout:write("Aborting crawling.\n")
io.stdout:flush()
killgrab = true
end
read_file = function(file)
local f = assert(io.open(file, "rb"))
local data = f:read("*all")
f:close()
return data
end
processed = function(url)
if downloaded[url] or addedtolist[url] then
return true
end
return false
end
discover_item = function(target, item)
if item ~= item_name and not target[item] then
--print("discovered", item)
target[item] = true
return true
end
return false
end
query_param = function(url, name)
local query = string.match(url, "%?(.+)$")
if not query then
return nil
end
for key, value in string.gmatch(query, "([^&=]+)=?([^&]*)") do
if key == name then
return urlparse.unescape(value)
end
end
return nil
end
find_item = function(url)
url = urlparse.unescape(url)
for pattern, type_ in pairs(item_patterns) do
local value = string.match(url, pattern)
if value then
return {
["type"]=type_,
["value"]=value
}
end
end
end
set_item = function(url)
local found = find_item(url)
if found then
local new_item_type = found["type"]
local new_item_value = found["value"]
local new_item_name = new_item_type .. ":" .. new_item_value
if new_item_name ~= item_name then
item_value = new_item_value
item_type = new_item_type
ids = {}
ids[item_value] = true
abortgrab = false
tries = 0
retry_url = false
item_name = new_item_name
print("Archiving item " .. item_name)
end
end
end
allowed = function(url, body_data)
if not string.match(url, "^https?://") then
return false
end
if not (
string.match(url, "^https?://www%.blice%.co%.kr/")
or string.match(url, "^https?://blice%.co%.kr/")
or string.match(url, "^https?://cds%.blice%.co%.kr/")
) then
return false
end
local found = find_item(url)
if found then
local found_item_name = found["type"] .. ":" .. found["value"]
if found_item_name ~= item_name then
discover_item(discovered_items, found_item_name)
return false
end
return true
end
for _, pattern in ipairs({
"^https?://[^/]*blice%.co%.kr/[mw][we]?b?/comment%.kt%?",
"^https?://[^/]*blice%.co%.kr/[mw][we]?b?/other_more_list%.kt%?",
"^https?://[^/]*blice%.co%.kr/web/readMoreTimesList_ajax%?",
"^https?://[^/]*blice%.co%.kr/mw/readMoreTimesList_ajax%?",
"^https?://[^/]*blice%.co%.kr/web/viewer_readMoreTimesList_ajax%?",
"^https?://[^/]*blice%.co%.kr/web/commentList_ajax%?",
"^https?://[^/]*blice%.co%.kr/web/event/commentList_ajax%?",
"^https?://[^/]*blice%.co%.kr/[mw][we]?b?/commentListDetail%.kt%?"
}) do
if string.match(url, pattern) then
for _, name in ipairs({
"novelId",
"novel_no",
"novelSeq",
"timesId",
"novelTimesSeq",
"exceptionalEventSeq",
"eventSeq"
}) do
if ids[query_param(url, name)] then
return true
end
end
end
end
if body_data then
return true
end
if item_type == "event"
and (
string.match(url, "^https?://[^/]*blice%.co%.kr/event/more%.kt%?")
or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?")
) then
return true
end
return false
end
wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
return false
end
decode_codepoint = function(newurl)
newurl = string.gsub(
newurl, "\\[uU]([0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F])",
function (s)
return utf8.char(tonumber(s, 16))
end
)
return newurl
end
wget.callbacks.get_urls = function(file, url, is_css, iri)
local urls = {}
downloaded[url] = true
set_item(url)
if abortgrab then
return {}
end
local function fix_case(newurl)
if not string.match(newurl, "^https?://[^/]") then
return newurl
end
if string.match(newurl, "^https?://[^/]+$") then
newurl = newurl .. "/"
end
local a, b = string.match(newurl, "^(https?://[^/]+/)(.*)$")
return string.lower(a) .. b
end
local function check(newurl, headers, body_data)
if not string.match(newurl, "^https?://") then
return nil
end
newurl = decode_codepoint(newurl)
if string.match(newurl, "[%s\\]") then
return nil
end
newurl = fix_case(newurl)
if not body_data
and string.match(newurl, "^https?://www%.blice%.co%.kr/")
and not string.match(newurl, "_ajax%?")
and not string.match(newurl, "/api/") then
newurl = string.gsub(newurl, "^https?://www%.blice%.co%.kr/", "https://blice.co.kr/")
end
local url = string.match(newurl, "^([^#]+)")
local url_ = url
while string.match(url_, "&") do
url_ = string.gsub(url_, "&", "&")
end
local method = body_data and "POST" or "GET"
local key = method .. "\0" .. url_ .. "\0" .. tostring(body_data)
if not processed(key)
and (body_data or not processed(url_))
and allowed(url_, body_data) then
local url_data = {
url=url_,
headers=headers or {}
}
if body_data then
url_data["body_data"] = body_data
url_data["method"] = method
end
table.insert(urls, url_data)
addedtolist[key] = true
if not body_data then
addedtolist[url_] = true
addedtolist[url] = true
end
return true
end
end
local function checknewurl(newurl)
newurl = decode_codepoint(newurl)
if string.match(newurl, "['\"><]") then
return nil
end
if string.match(newurl, "^https?:////") then
check(string.gsub(newurl, ":////", "://"))
elseif string.match(newurl, "^https?://") then
check(newurl)
elseif string.match(newurl, "^https?:\\/\\?/") then
check(string.gsub(newurl, "\\", ""))
elseif string.match(newurl, "^\\/\\/") then
checknewurl(string.gsub(newurl, "\\", ""))
elseif string.match(newurl, "^//") then
check(urlparse.absolute(url, newurl))
elseif string.match(newurl, "^\\/") then
checknewurl(string.gsub(newurl, "\\", ""))
elseif string.match(newurl, "^/") then
check(urlparse.absolute(url, newurl))
elseif string.match(newurl, "^%.%./") then
if string.match(url, "^https?://[^/]+/[^/]+/") then
check(urlparse.absolute(url, newurl))
else
checknewurl(string.match(newurl, "^%.%.(/.+)$"))
end
elseif string.match(newurl, "^%./") then
check(urlparse.absolute(url, newurl))
end
end
local function checknewshorturl(newurl)
newurl = decode_codepoint(newurl)
if string.match(newurl, "^%?") then
check(urlparse.absolute(url, newurl))
elseif not (
string.match(newurl, "^https?:\\?/\\?//?/?")
or string.match(newurl, "^[/\\]")
or string.match(newurl, "^%./")
or string.match(newurl, "^[jJ]ava[sS]cript:")
or string.match(newurl, "^[mM]ail[tT]o:")
or string.match(newurl, "^data:")
or string.match(newurl, "^%${")
) then
check(urlparse.absolute(url, newurl))
end
end
local function check_media(data)
local values = {data}
if type(data) == "table" then
for _, key in ipairs({
"epubCoverUrl",
"epubUrl",
"filePath1",
"filePath2",
"imagePath1",
"imagePath2",
"profileImgPath",
"sticonUrl",
"timesCoverImagePath",
"timesFilePath",
"userImage",
"workImageWideUrl",
"workThumbUrl"
}) do
if data[key] then
table.insert(values, data[key])
end
end
if type(data["webtoonImages"]) == "table" then
for _, value in ipairs(data["webtoonImages"]) do
table.insert(values, value)
end
end
for _, pair in ipairs({
{"epubSavePath", "epubNewName"},
{"epubCoverSavePath", "epubCoverNewName"}
}) do
local save_path = data[pair[1]]
local new_name = data[pair[2]]
if type(save_path) == "string" and type(new_name) == "string" then
save_path = string.gsub(save_path, "^/nas/noveltimes/epub/", "/epub/noveltimes/")
if string.match(save_path, "^/epub/noveltimes/") then
table.insert(values, save_path .. new_name)
end
end
end
end
for _, value in ipairs(values) do
if type(value) == "string" then
if string.match(value, "^https?://") then
check(value)
elseif string.match(value, "^/") then
check("https://cds.blice.co.kr/download?file=" .. value)
end
end
end
end
local function check_xmlhttprequest(url, referer, body)
local headers = {
["Content-Type"]="application/json; charset=UTF-8",
["X-Requested-With"]="XMLHttpRequest",
["Referer"]=referer
}
check(string.match(url, "^([^?]+)"), headers, body)
check(url, headers, body)
end
local function check_web_mw(path_query, web_extra, mw_extra)
web_extra = web_extra or ""
if not mw_extra then
mw_extra = web_extra
end
for _, data in ipairs({
{"web", web_extra},
{"mw", mw_extra}
}) do
check("https://blice.co.kr/" .. data[1] .. "/" .. path_query .. data[2])
end
end
local function check_readmore(endpoint, page_no, order_by, novel_type_seq)
local path = "readMoreTimesList_ajax"
local body_novel_no = item_value
if endpoint == "viewer" then
endpoint = "web"
path = "viewer_readMoreTimesList_ajax"
order_by = "recent"
body_novel_no = '"' .. item_value .. '"'
end
local url = "https://blice.co.kr/" .. endpoint .. "/" .. path
.. "?novel_no=" .. item_value
.. "&orderBy=" .. order_by
.. "&pageNo=" .. tostring(page_no)
local body = '{"novel_no":' .. body_novel_no
if novel_type_seq then
url = url .. "&novelTypeSeq=" .. novel_type_seq
body = body .. ',"novelTypeSeq":' .. novel_type_seq
end
body = body
.. ',"orderBy":"' .. order_by .. '"'
.. ',"pageNo":' .. tostring(page_no)
if endpoint == "mw" or path == "viewer_readMoreTimesList_ajax" then
body = body .. ',"row":20'
url = url .. "&row=20"
end
body = body .. "}"
check_xmlhttprequest(
url,
"https://blice.co.kr/" .. endpoint .. "/detail.kt?novelId=" .. item_value,
body
)
end
local function check_comment_ajax(novel_seq, times_seq, comment_type, page_no)
local referer = "https://blice.co.kr/web/comment.kt?novelId=" .. novel_seq
.. "&commentType=" .. comment_type
if times_seq ~= "0" then
referer = "https://blice.co.kr/web/comment.kt?timesId=" .. times_seq
.. "&novelId=" .. novel_seq
.. "&commentType=" .. comment_type
end
local body = '{"novelSeq":' .. novel_seq
.. ',"novelTimesSeq":' .. times_seq
.. ',"commentType":"' .. comment_type .. '"'
.. ',"orderBy":"recent"'
.. ',"pageNo":' .. tostring(page_no)
.. "}"
check_xmlhttprequest(
"https://blice.co.kr/web/commentList_ajax?novelSeq=" .. novel_seq
.. "&novelTimesSeq=" .. times_seq
.. "&commentType=" .. comment_type
.. "&orderBy=recent"
.. "&pageNo=" .. tostring(page_no),
referer,
body
)
end
if status_code == 200
and allowed(url)
and (
string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+%.kt")
or string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+_ajax%?")
or string.match(url, "^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?")
or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?")
) then
local html = read_file(file)
if string.match(url, "^https?://blice%.co%.kr/web/detail%.kt%?")
and query_param(url, "novelId") == item_value then
check_web_mw(
"detail.kt?novelId=" .. item_value,
"&isPrologTimes=false&voucherXCnt=0&voucherYCnt=0&voucherY1Cnt=0&voucherY3Cnt=0&voucherLCnt=0&voucherSCnt=0"
.. "&viewVoucherRentCnt=0&viewVoucherHaveCnt=0&needGaLog=true&isLogin=false"
.. "&prevUrl=http%3A%2F%2Fblice.co.kr%2Fweb%2Fviewer.kt",
""
)
check_web_mw("comment.kt?novelId=" .. item_value .. "&commentType=novel", "", "&quickViewYn=N")
for _, data in ipairs({
{"web", "seq"},
{"web", "recent"},
{"web", "regDate"},
{"mw", "seq"},
{"mw", "recent"}
}) do
check_readmore(data[1], 1, data[2])
end
local novel_type_seq = string.match(html, "novelTypeSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)")
or string.match(html, "novelTypeSeq=([0-9]+)")
if novel_type_seq then
for _, order_by in ipairs({"recent", "regDate"}) do
check_readmore("web", 1, order_by, novel_type_seq)
end
end
check_readmore("viewer", 1)
check_comment_ajax(item_value, "0", "novel", 1)
elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?")
and query_param(url, "timesId") == item_value then
local times_type = query_param(url, "timesType")
if times_type == "" then
times_type = nil
end
times_type = times_type
or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)")
or string.match(html, "timesType=([a-zA-Z]+)")
if not times_type
or not string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)") then
return urls
end
local viewer_path_query = "viewer.kt?timesId=" .. item_value .. "&timesType=" .. times_type
check_web_mw(viewer_path_query, "&quickViewYn=N")
check_web_mw(viewer_path_query, "&quickViewYn=")
check_web_mw(viewer_path_query)
check("https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value)
check_xmlhttprequest(
"https://blice.co.kr/api/novel/v1/viewerAjax?timesId=" .. item_value .. "&timesType=" .. times_type .. "&quickViewYn=N",
"https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value .. "&timesType=" .. times_type .. "&quickViewYn=N",
'{"timesId":' .. item_value .. ',"timesType":"' .. times_type .. '","quickViewYn":"N"}'
)
elseif string.match(url, "^https?://blice%.co%.kr/event/eventPage%.kt%?")
and query_param(url, "eventSeq") == item_value then
check("https://blice.co.kr/mw/event/eventPage.kt?eventSeq=" .. item_value)
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1")
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1&exceptionalEventSeq=" .. item_value .. "&pageNo=1")
elseif string.match(url, "^https?://blice%.co%.kr/web/documents/detail%.kt%?")
and query_param(url, "documentSeq") == item_value then
check_web_mw("documents/detail.kt?documentSeq=" .. item_value, "&start=1", "")
elseif string.match(url, "^https?://blice%.co%.kr/mw/retrieveSpecificUserProfile%.kt%?")
and query_param(url, "userNicknameSeq") == item_value then
if query_param(url, "userType") == "AUTHOR_PEN" then
local penname_id = query_param(url, "userPennameSeq")
check_xmlhttprequest(
"https://blice.co.kr/web/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userPennameSeq=" .. penname_id .. "&userNicknameSeq=" .. item_value,
"https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. penname_id,
'{"userType":"AUTHOR_PEN","userPennameSeq":"' .. penname_id .. '","userNicknameSeq":"' .. item_value .. '","adultViewYn":1,"onlyPenYn":"Y"}'
)
else
for _, user_type in ipairs({
"USER",
"AUTHOR_NICK"
}) do
check_xmlhttprequest(
"https://blice.co.kr/web/retrieveSpecificUserProfile.kt?userType=" .. user_type .. "&userNicknameSeq=" .. item_value,
"https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item_value,
'{"userType":"' .. user_type .. '","userNicknameSeq":"' .. item_value .. '"}'
)
end
end
end
if string.match(url, "/readMoreTimesList_ajax%?")
or string.match(url, "/viewer_readMoreTimesList_ajax%?") then
local response = cjson.decode(html)["response"]
local count = 0
local times_list = response["timesResList"]
if string.match(url, "/mw/")
or string.match(url, "/viewer_readMoreTimesList_ajax%?") then
times_list = response
end
for _, item in ipairs(times_list) do
count = count + 1
check("https://blice.co.kr/web/viewer.kt?timesId=" .. item["novelTimesSeq"])
check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
check_media(item)
end
local page_no = tonumber(query_param(url, "pageNo"))
local order_by = query_param(url, "orderBy")
local novel_type_seq = query_param(url, "novelTypeSeq")
local is_mw = string.match(url, "/mw/")
local is_viewer = string.match(url, "/viewer_readMoreTimesList_ajax%?")
local readmore_endpoint = "web"
if is_mw then
readmore_endpoint = "mw"
elseif is_viewer then
readmore_endpoint = "viewer"
end
if not is_mw and not is_viewer then
for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do
new_page = tonumber(new_page)
if new_page > page_no then
check_readmore("web", new_page, order_by, novel_type_seq)
end
end
end
if count > 0 then
if readmore_endpoint == "mw" then
check_readmore(readmore_endpoint, page_no + 1, order_by)
elseif readmore_endpoint == "viewer" then
check_readmore(readmore_endpoint, page_no + 1)
else
check_readmore(readmore_endpoint, page_no + 1, order_by, novel_type_seq)
end
end
elseif string.match(url, "/event/commentList_ajax%?") then
local response = cjson.decode(html)["response"]
local count = 0
for _, item in ipairs(response["eventCommentResList"]) do
count = count + 1
check_media(item)
end
local page_no = tonumber(query_param(url, "pageNo"))
local event_seq = query_param(url, "exceptionalEventSeq")
for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)") do
new_page = tonumber(new_page)
if new_page > page_no then
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(new_page))
end
end
if count >= 30 then
check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(page_no + 1))
end
elseif string.match(url, "/event/card%?") then
local count = 0
for _, item in ipairs(cjson.decode(html)["response"]["workList"]) do
count = count + 1
check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
check_media(item)
end
if count > 0 then
check("https://blice.co.kr/event/card?cardListSeq=" .. query_param(url, "cardListSeq")
.. "&startNo=" .. tostring(tonumber(query_param(url, "startNo")) + 1))
end
elseif string.match(url, "/commentList_ajax%?") then
local response = cjson.decode(html)["response"]
local count = 0
local comment_type = query_param(url, "commentType")
local times_seq = query_param(url, "novelTimesSeq")
for _, item in ipairs(response["timesCommentResList"]) do
count = count + 1
check_media(item)
check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item["userNicknameSeq"])
check_web_mw(
"commentListDetail.kt?novelId=" .. item["novelSeq"] .. "&timesId=" .. item["novelTimesSeq"],
"&commentId=" .. item["timesCommentSeq"] .. "&commentType=" .. comment_type,
"&timesType=" .. comment_type
.. "&commentId=" .. item["timesCommentSeq"]
.. "&commentType=" .. comment_type
.. "&quickViewYn=N"
)
end
local page_no = tonumber(query_param(url, "pageNo"))
for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do
new_page = tonumber(new_page)
if new_page > page_no then
check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, new_page)
end
end
if count >= 30 then
check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, page_no + 1)
end
elseif string.match(url, "^https?://[^/]*blice%.co%.kr/web/retrieveSpecificUserProfile%.kt%?") then
local response = cjson.decode(html)
check_media(response["specificUserProfile"])
if query_param(url, "userType") ~= "USER" then
for _, item in ipairs(response["authorNovelList"]) do
check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y"
.. "&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. item["authorPenSeq"])
check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
check_media(item)
end
else
for _, item in ipairs(response["userDocumentsList"]) do
check("https://blice.co.kr/web/documents/detail.kt?documentSeq=" .. item["documentSeq"])
end
end
elseif string.match(url, "/api/novel/v1/viewerAjax%?") then
local response = cjson.decode(html)["response"]
check("https://blice.co.kr/web/detail.kt?novelId=" .. response["novelSeq"])
check_media(response)
for _, key in ipairs({"prev", "next"}) do
local adjacent = response[key]
if type(adjacent) == "table" then
check("https://blice.co.kr/web/viewer.kt?timesId=" .. adjacent["novelTimesSeq"])
end
end
else
for _, data in ipairs({
{"novelId=([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"},
{"data%-novelSeq=['\"]?([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"},
{"data%-first%-timesId=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"},
{"data%-times%-seq=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"},
{"data%-nickNameSeq=\"([0-9]+)\"", "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=%s"},
{"eventSeq=([0-9]+)", "https://blice.co.kr/event/eventPage.kt?eventSeq=%s"},
{"documentSeq=([0-9]+)", "https://blice.co.kr/web/documents/detail.kt?documentSeq=%s"},
{"noticeSeq=([0-9]+)", "https://blice.co.kr/web/support/noticeDetail.kt?noticeSeq=%s"}
}) do
for value in string.gmatch(html, data[1]) do
check(string.format(data[2], value))
end
end
for times_id in string.gmatch(html, "timesId=([0-9]+)") do
check("https://blice.co.kr/web/viewer.kt?timesId=" .. times_id)
end
if query_param(url, "novelId") == item_value
or query_param(url, "novel_no") == item_value
or query_param(url, "novelSeq") == item_value then
for penname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\"") do
check_web_mw("other_more_list.kt?novelId=" .. item_value .. "&pennameId=" .. penname_id)
end
for penname_id, nickname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\" data%-nickname%-seq=\"([0-9]+)\" data%-adult%-view%-yn=\"[^\"]*\"") do
check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y"
.. "&userType=AUTHOR_PEN&userNicknameSeq=" .. nickname_id .. "&userPennameSeq=" .. penname_id)
end
elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?") then
local novel_no = string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)")
if not novel_no then
return urls
end
check("https://blice.co.kr/web/detail.kt?novelId=" .. novel_no)
local times_type = query_param(url, "timesType")
if times_type == "" then
times_type = nil
end
times_type = times_type
or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)")
or string.match(html, "timesType=([a-zA-Z]+)")
if not times_type then
return urls
end
check_web_mw("comment.kt?timesId=" .. item_value .. "&novelId=" .. novel_no
.. "&commentType=" .. times_type, "", "&quickViewYn=N")
check("https://blice.co.kr/mw/comment.kt?timesId=" .. item_value
.. "&novelId=" .. novel_no
.. "&commentType=" .. times_type
.. "&quickViewYn=N&hback=%2Fmw%2Fviewer.kt%3FtimesId%3D" .. item_value
.. "%26timesType%3D" .. times_type
.. "%26quickViewYn%3DN")
check_comment_ajax(novel_no, item_value, times_type, 1)
elseif string.match(url, "^https?://blice%.co%.kr/event/")
or string.match(url, "^https?://blice%.co%.kr/mw/event/") then
for card_list_id in string.gmatch(html, "cardListSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)") do
check("https://blice.co.kr/event/card?cardListSeq=" .. card_list_id .. "&startNo=2")
end
end
for newurl in string.gmatch(string.gsub(html, "&[qQ][uU][oO][tT];", '"'), '([^"]+)') do
checknewurl(newurl)
end
for newurl in string.gmatch(string.gsub(html, "&#039;", "'"), "([^']+)") do
checknewurl(newurl)
end
for newurl in string.gmatch(html, "[^%-]href='([^']+)'") do
checknewshorturl(newurl)
end
for newurl in string.gmatch(html, '[^%-]href="([^"]+)"') do
checknewshorturl(newurl)
end
for newurl in string.gmatch(html, "[^%-]src='([^']+)'") do
checknewshorturl(newurl)
end
for newurl in string.gmatch(html, '[^%-]src="([^"]+)"') do
checknewshorturl(newurl)
end
for newurl in string.gmatch(html, ":%s*url%(([^%)]+)%)") do
newurl = string.gsub(newurl, "^['\"]", "")
newurl = string.gsub(newurl, "['\"]$", "")
checknewurl(newurl)
end
html = string.gsub(html, "&gt;", ">")
html = string.gsub(html, "&lt;", "<")
for newurl in string.gmatch(html, ">%s*([^<%s]+)") do
checknewurl(newurl)
end
end
end
return urls
end
wget.callbacks.write_to_warc = function(url, http_stat)
status_code = http_stat["statcode"]
set_item(url["url"])
url_count = url_count + 1
io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n")
io.stdout:flush()
logged_response = true
if not item_name then
error("No item name found.")
end
if http_stat["len"] == 0
and status_code < 300 then
retry_url = true
return false
end
if status_code ~= 200
and status_code ~= 302 then
retry_url = true
return false
end
if abortgrab then
print("Not writing to WARC.")
return false
end
retry_url = false
tries = 0
return true
end
wget.callbacks.httploop_result = function(url, err, http_stat)
status_code = http_stat["statcode"]
if not logged_response then
url_count = url_count + 1
io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n")
io.stdout:flush()
end
logged_response = false
if killgrab then
return wget.actions.ABORT
end
set_item(url["url"])
if not item_name then
error("No item name found.")
end
if abortgrab then
abort_item()
return wget.actions.EXIT
end
if status_code == 0 or retry_url then
io.stdout:write("Server returned bad response. ")
io.stdout:flush()
tries = tries + 1
local maxtries = 6
if tries > maxtries then
io.stdout:write(" Skipping.\n")
io.stdout:flush()
tries = 0
abort_item()
return wget.actions.EXIT
end
local sleep_time = math.random(
math.floor(math.pow(2, tries-0.5)),
math.floor(math.pow(2, tries))
)
io.stdout:write("Sleeping " .. sleep_time .. " seconds.\n")
io.stdout:flush()
os.execute("sleep " .. sleep_time)
return wget.actions.CONTINUE
else
if status_code == 200 or status_code == 206 then
if not seen_200[url["url"]] then
seen_200[url["url"]] = 0
end
seen_200[url["url"]] = seen_200[url["url"]] + 1
end
downloaded[url["url"]] = true
end
if status_code >= 300 and status_code <= 399 then
local newloc = urlparse.absolute(url["url"], http_stat["newloc"])
if processed(newloc) or not allowed(newloc) then
tries = 0
return wget.actions.EXIT
end
end
tries = 0
return wget.actions.NOTHING
end
wget.callbacks.finish = function(start_time, end_time, wall_time, numurls, total_downloaded_bytes, total_download_time)
local function submit_backfeed(items, key)
local tries = 0
local maxtries = 5
while tries < maxtries do
if killgrab then
return false
end
local body, code, headers, status = http.request(
"https://legacy-api.arpa.li/backfeed/legacy/" .. key,
items .. "\0"
)
if code == 200 and body ~= nil and cjson.decode(body)["status_code"] == 200 then
io.stdout:write(string.match(body, "^(.-)%s*$") .. "\n")
io.stdout:flush()
return nil
end
io.stdout:write("Failed to submit discovered URLs." .. tostring(code) .. tostring(body) .. "\n")
io.stdout:flush()
os.execute("sleep " .. math.floor(math.pow(2, tries)))
tries = tries + 1
end
kill_grab()
error()
end
local file = io.open(item_dir .. "/" .. warc_file_base .. "_bad-items.txt", "w")
for url, _ in pairs(bad_items) do
file:write(url .. "\n")
end
file:close()
for key, data in pairs({
["blice-tob75dmr5nkp494q"] = discovered_items
}) do
print("queuing for", string.match(key, "^(.+)%-") or key)
local items = nil
local count = 0
for item, _ in pairs(data) do
print("found item", item)
if items == nil then
items = item
else
items = items .. "\0" .. item
end
count = count + 1
if count == 1000 then
submit_backfeed(items, key)
items = nil
count = 0
end
end
submit_backfeed(items, key)
end
end
wget.callbacks.before_exit = function(exit_status, exit_status_string)
if killgrab then
return wget.exits.IO_FAIL
end
if abortgrab then
abort_item()
end
return exit_status
end