From 69b0038af582427a384654aec2349a70430e6c3f Mon Sep 17 00:00:00 2001 From: arkiver Date: Mon, 29 Jun 2026 04:23:36 +0200 Subject: [PATCH] Version 20260629.01. Initial. --- .gitignore | 5 + Dockerfile | 1 + LICENSE | 24 ++ README.md | 56 ++++ blice.lua | 917 ++++++++++++++++++++++++++++++++++++++++++++++++++++ pipeline.py | 374 +++++++++++++++++++++ 6 files changed, 1377 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 blice.lua create mode 100644 pipeline.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..829e598 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*~ +*.old +wget-at +STOP +data/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..586a4db --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +FROM atdr.meo.ws/archiveteam/grab-base:nss diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cf1ab25 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/README.md b/README.md new file mode 100644 index 0000000..c84c3e9 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# blice-grab + +More information about the archiving project can be found on the ArchiveTeam wiki: [Blice](https://wiki.archiveteam.org/index.php?title=Blice) + +## Donations + +Behind the scenes Archive Team has infrastructure to run the projects and process the data with. If you would like to help out with the costs of our infrastructure, a donation on our [Open Collective](https://opencollective.com/archiveteam) would be very welcome. + +## Setup instructions + +### General instructions + +Data integrity is very important in Archive Team projects. Please note the following important rules: + +* [Do not use proxies or VPNs](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior#Can_I_use_whatever_internet_access_for_the_Warrior?). +* Run the project using the either the Warrior or the project-specific Docker container as listed below. [Do not modify project code](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior#I'd_like_to_help_write_code_or_I_want_to_tweak_the_scripts_to_run_to_my_liking._Where_can_I_find_more_info?_Where_is_the_source_code_and_repository?). Compiling the project dependencies yourself is no longer supported. +* You can share your tracker nickname(s) across machine(s) you personally operate, but not with machines operated by other users. Nickname sharing makes it harder to inspect data if a problem arises. +* [Use clean internet connections](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior#Can_I_use_whatever_internet_access_for_the_Warrior?). +* Only x64-based machines are supported. [ARM (used on Raspberry Pi and Apple Silicon Macs) is not currently supported](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior#Can_I_run_the_Warrior_on_ARM_or_some_other_unusual_architecture?). +* See the [Archive Team Wiki](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior#Warrior_FAQ) for additional information. + +We strongly encourage you to join the IRC channel associated with this project in order to be informed about project updates and other important announcements, as well as to be reachable in the event of an issue. The Archive Team Wiki has [more information about IRC](https://wiki.archiveteam.org/index.php/Archiveteam:IRC). We can be found at hackint IRC [#archiveteam-bs](https://chat.hackint.org/?join=%23archiveteam-bs). + +**If you have any questions or issues during setup, please review the wiki pages or contact us on IRC for troubleshooting information.** + +### Running the project + +#### Archive Team Warrior (recommended for most users) + +This and other archiving projects can easily be run using the [Archive Team Warrior](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior) virtual machine. Follow the [instructions on the Archive Team wiki](https://wiki.archiveteam.org/index.php/ArchiveTeam_Warrior) for installing the Warrior, and from the web interface running at `http://localhost:8001/`, enter the nickname that you want to be shown as on the tracker. There is no registration, just pick a nickname you like. Then, select the `Blice` project in the Warrior interface. + +#### Project-specific Docker container (for more advanced users) + +Alternatively, more advanced users can also run projects using Docker. While users of the Warrior can switch between projects using a web interface, Docker containers are specific to each project. However, while the Warrior supports a maximum of 6 concurrent items, a Docker container supports a maximum of 20 concurrent items. The instructions below are a short overview. For more information and detailed explanations of the commands, follow the follow the [Docker instructions on the Archive Team wiki](https://wiki.archiveteam.org/index.php/Running_Archive_Team_Projects_with_Docker). + +It is advised to use [Watchtower](https://github.com/containrrr/watchtower) to automatically update the project container: + + docker run -d --name watchtower --restart=unless-stopped -v /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower --label-enable --cleanup --interval 3600 --include-restarting + +after which the project container can be run: + + docker run -d --name archiveteam --label=com.centurylinklabs.watchtower.enable=true --log-driver json-file --log-opt max-size=50m --restart=unless-stopped atdr.meo.ws/archiveteam/blice-grab --concurrent 1 YOURNICKHERE + +Be sure to replace `YOURNICKHERE` with the nickname that you want to be shown as on the tracker. There is no registration, just pick a nickname you like. + +### Issues in the code + +If you notice a bug and want to file a bug report, please use the GitHub issues tracker. + +Are you a developer? Help write code for us! Look at our [developer documentation](https://wiki.archiveteam.org/index.php?title=Dev) for details. + +### Other problems + +Have an issue not listed here? Join us on IRC and ask! We can be found at hackint IRC [#archiveteam-bs](https://chat.hackint.org/?join=%23archiveteam-bs). + + diff --git a/blice.lua b/blice.lua new file mode 100644 index 0000000..301b042 --- /dev/null +++ b/blice.lua @@ -0,0 +1,917 @@ +local urlparse = require("socket.url") +local http = require("socket.http") +local cjson = require("cjson") +local utf8 = require("utf8") + +local item_dir = os.getenv("item_dir") +local warc_file_base = os.getenv("warc_file_base") +local concurrency = tonumber(os.getenv("concurrency")) +local item_type = nil +local item_name = nil +local item_value = nil +local ids = {} + +local url_count = 0 +local tries = 0 +local downloaded = {} +local seen_200 = {} +local addedtolist = {} +local abortgrab = false +local killgrab = false +local logged_response = false + +local discovered_items = {} +local bad_items = {} + +local retry_url = false + +local item_patterns = { + ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/viewer%.kt%?.*timesId=([0-9]+)"]="times", + ["^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?.*timesId=([0-9]+)"]="times", + ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/detail%.kt%?.*novelId=([0-9]+)"]="novel", + ["^https?://[^/]*blice%.co%.kr/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event", + ["^https?://[^/]*blice%.co%.kr/mw/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event", + ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/documents/detail%.kt%?.*documentSeq=([0-9]+)"]="document", + ["^https?://[^/]*blice%.co%.kr/web/support/noticeDetail%.kt%?.*noticeSeq=([0-9]+)"]="notice", + ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/retrieveSpecificUserProfile%.kt%?.*userNicknameSeq=([0-9]+)"]="user", + ["^https?://(cds%.blice%.co%.kr/download%?file=.+)"]="asset" +} + +abort_item = function(item) + abortgrab = true + if not item then + item = item_name + end + if not bad_items[item] then + io.stdout:write("Aborting item " .. item .. ".\n") + io.stdout:flush() + bad_items[item] = true + end +end + +kill_grab = function(item) + io.stdout:write("Aborting crawling.\n") + io.stdout:flush() + killgrab = true +end + +read_file = function(file) + local f = assert(io.open(file, "rb")) + local data = f:read("*all") + f:close() + return data +end + +processed = function(url) + if downloaded[url] or addedtolist[url] then + return true + end + return false +end + +discover_item = function(target, item) + if item ~= item_name and not target[item] then +--print("discovered", item) + target[item] = true + return true + end + return false +end + +query_param = function(url, name) + local query = string.match(url, "%?(.+)$") + if not query then + return nil + end + for key, value in string.gmatch(query, "([^&=]+)=?([^&]*)") do + if key == name then + return urlparse.unescape(value) + end + end + return nil +end + +find_item = function(url) + url = urlparse.unescape(url) + for pattern, type_ in pairs(item_patterns) do + local value = string.match(url, pattern) + if value then + return { + ["type"]=type_, + ["value"]=value + } + end + end +end + +set_item = function(url) + local found = find_item(url) + if found then + local new_item_type = found["type"] + local new_item_value = found["value"] + local new_item_name = new_item_type .. ":" .. new_item_value + if new_item_name ~= item_name then + item_value = new_item_value + item_type = new_item_type + ids = {} + ids[item_value] = true + abortgrab = false + tries = 0 + retry_url = false + item_name = new_item_name + print("Archiving item " .. item_name) + end + end +end + +allowed = function(url, body_data) + if not string.match(url, "^https?://") then + return false + end + + if not ( + string.match(url, "^https?://www%.blice%.co%.kr/") + or string.match(url, "^https?://blice%.co%.kr/") + or string.match(url, "^https?://cds%.blice%.co%.kr/") + ) then + return false + end + + local found = find_item(url) + if found then + local found_item_name = found["type"] .. ":" .. found["value"] + if found_item_name ~= item_name then + discover_item(discovered_items, found_item_name) + return false + end + return true + end + + for _, pattern in ipairs({ + "^https?://[^/]*blice%.co%.kr/[mw][we]?b?/comment%.kt%?", + "^https?://[^/]*blice%.co%.kr/[mw][we]?b?/other_more_list%.kt%?", + "^https?://[^/]*blice%.co%.kr/web/readMoreTimesList_ajax%?", + "^https?://[^/]*blice%.co%.kr/mw/readMoreTimesList_ajax%?", + "^https?://[^/]*blice%.co%.kr/web/viewer_readMoreTimesList_ajax%?", + "^https?://[^/]*blice%.co%.kr/web/commentList_ajax%?", + "^https?://[^/]*blice%.co%.kr/web/event/commentList_ajax%?", + "^https?://[^/]*blice%.co%.kr/[mw][we]?b?/commentListDetail%.kt%?" + }) do + if string.match(url, pattern) then + for _, name in ipairs({ + "novelId", + "novel_no", + "novelSeq", + "timesId", + "novelTimesSeq", + "exceptionalEventSeq", + "eventSeq" + }) do + if ids[query_param(url, name)] then + return true + end + end + end + end + + if body_data then + return true + end + + if item_type == "event" + and ( + string.match(url, "^https?://[^/]*blice%.co%.kr/event/more%.kt%?") + or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?") + ) then + return true + end + + return false +end + +wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason) + return false +end + +decode_codepoint = function(newurl) + newurl = string.gsub( + newurl, "\\[uU]([0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F])", + function (s) + return utf8.char(tonumber(s, 16)) + end + ) + return newurl +end + +wget.callbacks.get_urls = function(file, url, is_css, iri) + local urls = {} + + downloaded[url] = true + set_item(url) + + if abortgrab then + return {} + end + + local function fix_case(newurl) + if not string.match(newurl, "^https?://[^/]") then + return newurl + end + if string.match(newurl, "^https?://[^/]+$") then + newurl = newurl .. "/" + end + local a, b = string.match(newurl, "^(https?://[^/]+/)(.*)$") + return string.lower(a) .. b + end + + local function check(newurl, headers, body_data) + if not string.match(newurl, "^https?://") then + return nil + end + newurl = decode_codepoint(newurl) + if string.match(newurl, "[%s\\]") then + return nil + end + newurl = fix_case(newurl) + if not body_data + and string.match(newurl, "^https?://www%.blice%.co%.kr/") + and not string.match(newurl, "_ajax%?") + and not string.match(newurl, "/api/") then + newurl = string.gsub(newurl, "^https?://www%.blice%.co%.kr/", "https://blice.co.kr/") + end + local url = string.match(newurl, "^([^#]+)") + local url_ = url + while string.match(url_, "&") do + url_ = string.gsub(url_, "&", "&") + end + local method = body_data and "POST" or "GET" + local key = method .. "\0" .. url_ .. "\0" .. tostring(body_data) + if not processed(key) + and (body_data or not processed(url_)) + and allowed(url_, body_data) then + local url_data = { + url=url_, + headers=headers or {} + } + if body_data then + url_data["body_data"] = body_data + url_data["method"] = method + end + table.insert(urls, url_data) + addedtolist[key] = true + if not body_data then + addedtolist[url_] = true + addedtolist[url] = true + end + return true + end + end + + local function checknewurl(newurl) + newurl = decode_codepoint(newurl) + if string.match(newurl, "['\"><]") then + return nil + end + if string.match(newurl, "^https?:////") then + check(string.gsub(newurl, ":////", "://")) + elseif string.match(newurl, "^https?://") then + check(newurl) + elseif string.match(newurl, "^https?:\\/\\?/") then + check(string.gsub(newurl, "\\", "")) + elseif string.match(newurl, "^\\/\\/") then + checknewurl(string.gsub(newurl, "\\", "")) + elseif string.match(newurl, "^//") then + check(urlparse.absolute(url, newurl)) + elseif string.match(newurl, "^\\/") then + checknewurl(string.gsub(newurl, "\\", "")) + elseif string.match(newurl, "^/") then + check(urlparse.absolute(url, newurl)) + elseif string.match(newurl, "^%.%./") then + if string.match(url, "^https?://[^/]+/[^/]+/") then + check(urlparse.absolute(url, newurl)) + else + checknewurl(string.match(newurl, "^%.%.(/.+)$")) + end + elseif string.match(newurl, "^%./") then + check(urlparse.absolute(url, newurl)) + end + end + + local function checknewshorturl(newurl) + newurl = decode_codepoint(newurl) + if string.match(newurl, "^%?") then + check(urlparse.absolute(url, newurl)) + elseif not ( + string.match(newurl, "^https?:\\?/\\?//?/?") + or string.match(newurl, "^[/\\]") + or string.match(newurl, "^%./") + or string.match(newurl, "^[jJ]ava[sS]cript:") + or string.match(newurl, "^[mM]ail[tT]o:") + or string.match(newurl, "^data:") + or string.match(newurl, "^%${") + ) then + check(urlparse.absolute(url, newurl)) + end + end + + local function check_media(data) + local values = {data} + if type(data) == "table" then + for _, key in ipairs({ + "epubCoverUrl", + "epubUrl", + "filePath1", + "filePath2", + "imagePath1", + "imagePath2", + "profileImgPath", + "sticonUrl", + "timesCoverImagePath", + "timesFilePath", + "userImage", + "workImageWideUrl", + "workThumbUrl" + }) do + if data[key] then + table.insert(values, data[key]) + end + end + if type(data["webtoonImages"]) == "table" then + for _, value in ipairs(data["webtoonImages"]) do + table.insert(values, value) + end + end + for _, pair in ipairs({ + {"epubSavePath", "epubNewName"}, + {"epubCoverSavePath", "epubCoverNewName"} + }) do + local save_path = data[pair[1]] + local new_name = data[pair[2]] + if type(save_path) == "string" and type(new_name) == "string" then + save_path = string.gsub(save_path, "^/nas/noveltimes/epub/", "/epub/noveltimes/") + if string.match(save_path, "^/epub/noveltimes/") then + table.insert(values, save_path .. new_name) + end + end + end + end + for _, value in ipairs(values) do + if type(value) == "string" then + if string.match(value, "^https?://") then + check(value) + elseif string.match(value, "^/") then + check("https://cds.blice.co.kr/download?file=" .. value) + end + end + end + end + + local function check_xmlhttprequest(url, referer, body) + local headers = { + ["Content-Type"]="application/json; charset=UTF-8", + ["X-Requested-With"]="XMLHttpRequest", + ["Referer"]=referer + } + check(string.match(url, "^([^?]+)"), headers, body) + check(url, headers, body) + end + + local function check_web_mw(path_query, web_extra, mw_extra) + web_extra = web_extra or "" + if not mw_extra then + mw_extra = web_extra + end + for _, data in ipairs({ + {"web", web_extra}, + {"mw", mw_extra} + }) do + check("https://blice.co.kr/" .. data[1] .. "/" .. path_query .. data[2]) + end + end + + local function check_readmore(endpoint, page_no, order_by, novel_type_seq) + local path = "readMoreTimesList_ajax" + local body_novel_no = item_value + if endpoint == "viewer" then + endpoint = "web" + path = "viewer_readMoreTimesList_ajax" + order_by = "recent" + body_novel_no = '"' .. item_value .. '"' + end + local url = "https://blice.co.kr/" .. endpoint .. "/" .. path + .. "?novel_no=" .. item_value + .. "&orderBy=" .. order_by + .. "&pageNo=" .. tostring(page_no) + local body = '{"novel_no":' .. body_novel_no + if novel_type_seq then + url = url .. "&novelTypeSeq=" .. novel_type_seq + body = body .. ',"novelTypeSeq":' .. novel_type_seq + end + body = body + .. ',"orderBy":"' .. order_by .. '"' + .. ',"pageNo":' .. tostring(page_no) + if endpoint == "mw" or path == "viewer_readMoreTimesList_ajax" then + body = body .. ',"row":20' + url = url .. "&row=20" + end + body = body .. "}" + check_xmlhttprequest( + url, + "https://blice.co.kr/" .. endpoint .. "/detail.kt?novelId=" .. item_value, + body + ) + end + + local function check_comment_ajax(novel_seq, times_seq, comment_type, page_no) + local referer = "https://blice.co.kr/web/comment.kt?novelId=" .. novel_seq + .. "&commentType=" .. comment_type + if times_seq ~= "0" then + referer = "https://blice.co.kr/web/comment.kt?timesId=" .. times_seq + .. "&novelId=" .. novel_seq + .. "&commentType=" .. comment_type + end + local body = '{"novelSeq":' .. novel_seq + .. ',"novelTimesSeq":' .. times_seq + .. ',"commentType":"' .. comment_type .. '"' + .. ',"orderBy":"recent"' + .. ',"pageNo":' .. tostring(page_no) + .. "}" + check_xmlhttprequest( + "https://blice.co.kr/web/commentList_ajax?novelSeq=" .. novel_seq + .. "&novelTimesSeq=" .. times_seq + .. "&commentType=" .. comment_type + .. "&orderBy=recent" + .. "&pageNo=" .. tostring(page_no), + referer, + body + ) + end + + if status_code == 200 + and allowed(url) + and ( + string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+%.kt") + or string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+_ajax%?") + or string.match(url, "^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?") + or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?") + ) then + local html = read_file(file) + if string.match(url, "^https?://blice%.co%.kr/web/detail%.kt%?") + and query_param(url, "novelId") == item_value then + check_web_mw( + "detail.kt?novelId=" .. item_value, + "&isPrologTimes=false&voucherXCnt=0&voucherYCnt=0&voucherY1Cnt=0&voucherY3Cnt=0&voucherLCnt=0&voucherSCnt=0" + .. "&viewVoucherRentCnt=0&viewVoucherHaveCnt=0&needGaLog=true&isLogin=false" + .. "&prevUrl=http%3A%2F%2Fblice.co.kr%2Fweb%2Fviewer.kt", + "" + ) + check_web_mw("comment.kt?novelId=" .. item_value .. "&commentType=novel", "", "&quickViewYn=N") + for _, data in ipairs({ + {"web", "seq"}, + {"web", "recent"}, + {"web", "regDate"}, + {"mw", "seq"}, + {"mw", "recent"} + }) do + check_readmore(data[1], 1, data[2]) + end + local novel_type_seq = string.match(html, "novelTypeSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)") + or string.match(html, "novelTypeSeq=([0-9]+)") + if novel_type_seq then + for _, order_by in ipairs({"recent", "regDate"}) do + check_readmore("web", 1, order_by, novel_type_seq) + end + end + check_readmore("viewer", 1) + check_comment_ajax(item_value, "0", "novel", 1) + elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?") + and query_param(url, "timesId") == item_value then + local times_type = query_param(url, "timesType") + if times_type == "" then + times_type = nil + end + times_type = times_type + or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)") + or string.match(html, "timesType=([a-zA-Z]+)") + if not times_type + or not string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)") then + return urls + end + local viewer_path_query = "viewer.kt?timesId=" .. item_value .. "×Type=" .. times_type + check_web_mw(viewer_path_query, "&quickViewYn=N") + check_web_mw(viewer_path_query, "&quickViewYn=") + check_web_mw(viewer_path_query) + check("https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value) + check_xmlhttprequest( + "https://blice.co.kr/api/novel/v1/viewerAjax?timesId=" .. item_value .. "×Type=" .. times_type .. "&quickViewYn=N", + "https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value .. "×Type=" .. times_type .. "&quickViewYn=N", + '{"timesId":' .. item_value .. ',"timesType":"' .. times_type .. '","quickViewYn":"N"}' + ) + elseif string.match(url, "^https?://blice%.co%.kr/event/eventPage%.kt%?") + and query_param(url, "eventSeq") == item_value then + check("https://blice.co.kr/mw/event/eventPage.kt?eventSeq=" .. item_value) + check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1") + check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1&exceptionalEventSeq=" .. item_value .. "&pageNo=1") + elseif string.match(url, "^https?://blice%.co%.kr/web/documents/detail%.kt%?") + and query_param(url, "documentSeq") == item_value then + check_web_mw("documents/detail.kt?documentSeq=" .. item_value, "&start=1", "") + elseif string.match(url, "^https?://blice%.co%.kr/mw/retrieveSpecificUserProfile%.kt%?") + and query_param(url, "userNicknameSeq") == item_value then + if query_param(url, "userType") == "AUTHOR_PEN" then + local penname_id = query_param(url, "userPennameSeq") + check_xmlhttprequest( + "https://blice.co.kr/web/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userPennameSeq=" .. penname_id .. "&userNicknameSeq=" .. item_value, + "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. penname_id, + '{"userType":"AUTHOR_PEN","userPennameSeq":"' .. penname_id .. '","userNicknameSeq":"' .. item_value .. '","adultViewYn":1,"onlyPenYn":"Y"}' + ) + else + for _, user_type in ipairs({ + "USER", + "AUTHOR_NICK" + }) do + check_xmlhttprequest( + "https://blice.co.kr/web/retrieveSpecificUserProfile.kt?userType=" .. user_type .. "&userNicknameSeq=" .. item_value, + "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item_value, + '{"userType":"' .. user_type .. '","userNicknameSeq":"' .. item_value .. '"}' + ) + end + end + end + if string.match(url, "/readMoreTimesList_ajax%?") + or string.match(url, "/viewer_readMoreTimesList_ajax%?") then + local response = cjson.decode(html)["response"] + local count = 0 + local times_list = response["timesResList"] + if string.match(url, "/mw/") + or string.match(url, "/viewer_readMoreTimesList_ajax%?") then + times_list = response + end + for _, item in ipairs(times_list) do + count = count + 1 + check("https://blice.co.kr/web/viewer.kt?timesId=" .. item["novelTimesSeq"]) + check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"]) + check_media(item) + end + local page_no = tonumber(query_param(url, "pageNo")) + local order_by = query_param(url, "orderBy") + local novel_type_seq = query_param(url, "novelTypeSeq") + local is_mw = string.match(url, "/mw/") + local is_viewer = string.match(url, "/viewer_readMoreTimesList_ajax%?") + local readmore_endpoint = "web" + if is_mw then + readmore_endpoint = "mw" + elseif is_viewer then + readmore_endpoint = "viewer" + end + if not is_mw and not is_viewer then + for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do + new_page = tonumber(new_page) + if new_page > page_no then + check_readmore("web", new_page, order_by, novel_type_seq) + end + end + end + if count > 0 then + if readmore_endpoint == "mw" then + check_readmore(readmore_endpoint, page_no + 1, order_by) + elseif readmore_endpoint == "viewer" then + check_readmore(readmore_endpoint, page_no + 1) + else + check_readmore(readmore_endpoint, page_no + 1, order_by, novel_type_seq) + end + end + elseif string.match(url, "/event/commentList_ajax%?") then + local response = cjson.decode(html)["response"] + local count = 0 + for _, item in ipairs(response["eventCommentResList"]) do + count = count + 1 + check_media(item) + end + local page_no = tonumber(query_param(url, "pageNo")) + local event_seq = query_param(url, "exceptionalEventSeq") + for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)") do + new_page = tonumber(new_page) + if new_page > page_no then + check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(new_page)) + end + end + if count >= 30 then + check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(page_no + 1)) + end + elseif string.match(url, "/event/card%?") then + local count = 0 + for _, item in ipairs(cjson.decode(html)["response"]["workList"]) do + count = count + 1 + check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"]) + check_media(item) + end + if count > 0 then + check("https://blice.co.kr/event/card?cardListSeq=" .. query_param(url, "cardListSeq") + .. "&startNo=" .. tostring(tonumber(query_param(url, "startNo")) + 1)) + end + elseif string.match(url, "/commentList_ajax%?") then + local response = cjson.decode(html)["response"] + local count = 0 + local comment_type = query_param(url, "commentType") + local times_seq = query_param(url, "novelTimesSeq") + for _, item in ipairs(response["timesCommentResList"]) do + count = count + 1 + check_media(item) + check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item["userNicknameSeq"]) + check_web_mw( + "commentListDetail.kt?novelId=" .. item["novelSeq"] .. "×Id=" .. item["novelTimesSeq"], + "&commentId=" .. item["timesCommentSeq"] .. "&commentType=" .. comment_type, + "×Type=" .. comment_type + .. "&commentId=" .. item["timesCommentSeq"] + .. "&commentType=" .. comment_type + .. "&quickViewYn=N" + ) + end + local page_no = tonumber(query_param(url, "pageNo")) + for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do + new_page = tonumber(new_page) + if new_page > page_no then + check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, new_page) + end + end + if count >= 30 then + check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, page_no + 1) + end + elseif string.match(url, "^https?://[^/]*blice%.co%.kr/web/retrieveSpecificUserProfile%.kt%?") then + local response = cjson.decode(html) + check_media(response["specificUserProfile"]) + if query_param(url, "userType") ~= "USER" then + for _, item in ipairs(response["authorNovelList"]) do + check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y" + .. "&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. item["authorPenSeq"]) + check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"]) + check_media(item) + end + else + for _, item in ipairs(response["userDocumentsList"]) do + check("https://blice.co.kr/web/documents/detail.kt?documentSeq=" .. item["documentSeq"]) + end + end + elseif string.match(url, "/api/novel/v1/viewerAjax%?") then + local response = cjson.decode(html)["response"] + check("https://blice.co.kr/web/detail.kt?novelId=" .. response["novelSeq"]) + check_media(response) + for _, key in ipairs({"prev", "next"}) do + local adjacent = response[key] + if type(adjacent) == "table" then + check("https://blice.co.kr/web/viewer.kt?timesId=" .. adjacent["novelTimesSeq"]) + end + end + else + for _, data in ipairs({ + {"novelId=([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"}, + {"data%-novelSeq=['\"]?([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"}, + {"data%-first%-timesId=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"}, + {"data%-times%-seq=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"}, + {"data%-nickNameSeq=\"([0-9]+)\"", "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=%s"}, + {"eventSeq=([0-9]+)", "https://blice.co.kr/event/eventPage.kt?eventSeq=%s"}, + {"documentSeq=([0-9]+)", "https://blice.co.kr/web/documents/detail.kt?documentSeq=%s"}, + {"noticeSeq=([0-9]+)", "https://blice.co.kr/web/support/noticeDetail.kt?noticeSeq=%s"} + }) do + for value in string.gmatch(html, data[1]) do + check(string.format(data[2], value)) + end + end + for times_id in string.gmatch(html, "timesId=([0-9]+)") do + check("https://blice.co.kr/web/viewer.kt?timesId=" .. times_id) + end + if query_param(url, "novelId") == item_value + or query_param(url, "novel_no") == item_value + or query_param(url, "novelSeq") == item_value then + for penname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\"") do + check_web_mw("other_more_list.kt?novelId=" .. item_value .. "&pennameId=" .. penname_id) + end + for penname_id, nickname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\" data%-nickname%-seq=\"([0-9]+)\" data%-adult%-view%-yn=\"[^\"]*\"") do + check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y" + .. "&userType=AUTHOR_PEN&userNicknameSeq=" .. nickname_id .. "&userPennameSeq=" .. penname_id) + end + elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?") then + local novel_no = string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)") + if not novel_no then + return urls + end + check("https://blice.co.kr/web/detail.kt?novelId=" .. novel_no) + local times_type = query_param(url, "timesType") + if times_type == "" then + times_type = nil + end + times_type = times_type + or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)") + or string.match(html, "timesType=([a-zA-Z]+)") + if not times_type then + return urls + end + check_web_mw("comment.kt?timesId=" .. item_value .. "&novelId=" .. novel_no + .. "&commentType=" .. times_type, "", "&quickViewYn=N") + check("https://blice.co.kr/mw/comment.kt?timesId=" .. item_value + .. "&novelId=" .. novel_no + .. "&commentType=" .. times_type + .. "&quickViewYn=N&hback=%2Fmw%2Fviewer.kt%3FtimesId%3D" .. item_value + .. "%26timesType%3D" .. times_type + .. "%26quickViewYn%3DN") + check_comment_ajax(novel_no, item_value, times_type, 1) + elseif string.match(url, "^https?://blice%.co%.kr/event/") + or string.match(url, "^https?://blice%.co%.kr/mw/event/") then + for card_list_id in string.gmatch(html, "cardListSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)") do + check("https://blice.co.kr/event/card?cardListSeq=" .. card_list_id .. "&startNo=2") + end + end + for newurl in string.gmatch(string.gsub(html, "&[qQ][uU][oO][tT];", '"'), '([^"]+)') do + checknewurl(newurl) + end + for newurl in string.gmatch(string.gsub(html, "'", "'"), "([^']+)") do + checknewurl(newurl) + end + for newurl in string.gmatch(html, "[^%-]href='([^']+)'") do + checknewshorturl(newurl) + end + for newurl in string.gmatch(html, '[^%-]href="([^"]+)"') do + checknewshorturl(newurl) + end + for newurl in string.gmatch(html, "[^%-]src='([^']+)'") do + checknewshorturl(newurl) + end + for newurl in string.gmatch(html, '[^%-]src="([^"]+)"') do + checknewshorturl(newurl) + end + for newurl in string.gmatch(html, ":%s*url%(([^%)]+)%)") do + newurl = string.gsub(newurl, "^['\"]", "") + newurl = string.gsub(newurl, "['\"]$", "") + checknewurl(newurl) + end + html = string.gsub(html, ">", ">") + html = string.gsub(html, "<", "<") + for newurl in string.gmatch(html, ">%s*([^<%s]+)") do + checknewurl(newurl) + end + end + end + + return urls +end + +wget.callbacks.write_to_warc = function(url, http_stat) + status_code = http_stat["statcode"] + set_item(url["url"]) + url_count = url_count + 1 + io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n") + io.stdout:flush() + logged_response = true + if not item_name then + error("No item name found.") + end + if http_stat["len"] == 0 + and status_code < 300 then + retry_url = true + return false + end + if status_code ~= 200 + and status_code ~= 302 then + retry_url = true + return false + end + if abortgrab then + print("Not writing to WARC.") + return false + end + retry_url = false + tries = 0 + return true +end + +wget.callbacks.httploop_result = function(url, err, http_stat) + status_code = http_stat["statcode"] + + if not logged_response then + url_count = url_count + 1 + io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n") + io.stdout:flush() + end + logged_response = false + + if killgrab then + return wget.actions.ABORT + end + + set_item(url["url"]) + if not item_name then + error("No item name found.") + end + + if abortgrab then + abort_item() + return wget.actions.EXIT + end + + if status_code == 0 or retry_url then + io.stdout:write("Server returned bad response. ") + io.stdout:flush() + tries = tries + 1 + local maxtries = 6 + if tries > maxtries then + io.stdout:write(" Skipping.\n") + io.stdout:flush() + tries = 0 + abort_item() + return wget.actions.EXIT + end + local sleep_time = math.random( + math.floor(math.pow(2, tries-0.5)), + math.floor(math.pow(2, tries)) + ) + io.stdout:write("Sleeping " .. sleep_time .. " seconds.\n") + io.stdout:flush() + os.execute("sleep " .. sleep_time) + return wget.actions.CONTINUE + else + if status_code == 200 or status_code == 206 then + if not seen_200[url["url"]] then + seen_200[url["url"]] = 0 + end + seen_200[url["url"]] = seen_200[url["url"]] + 1 + end + downloaded[url["url"]] = true + end + + if status_code >= 300 and status_code <= 399 then + local newloc = urlparse.absolute(url["url"], http_stat["newloc"]) + if processed(newloc) or not allowed(newloc) then + tries = 0 + return wget.actions.EXIT + end + end + + tries = 0 + + return wget.actions.NOTHING +end + +wget.callbacks.finish = function(start_time, end_time, wall_time, numurls, total_downloaded_bytes, total_download_time) + local function submit_backfeed(items, key) + local tries = 0 + local maxtries = 5 + while tries < maxtries do + if killgrab then + return false + end + local body, code, headers, status = http.request( + "https://legacy-api.arpa.li/backfeed/legacy/" .. key, + items .. "\0" + ) + if code == 200 and body ~= nil and cjson.decode(body)["status_code"] == 200 then + io.stdout:write(string.match(body, "^(.-)%s*$") .. "\n") + io.stdout:flush() + return nil + end + io.stdout:write("Failed to submit discovered URLs." .. tostring(code) .. tostring(body) .. "\n") + io.stdout:flush() + os.execute("sleep " .. math.floor(math.pow(2, tries))) + tries = tries + 1 + end + kill_grab() + error() + end + + local file = io.open(item_dir .. "/" .. warc_file_base .. "_bad-items.txt", "w") + for url, _ in pairs(bad_items) do + file:write(url .. "\n") + end + file:close() + for key, data in pairs({ + ["blice-tob75dmr5nkp494q"] = discovered_items + }) do + print("queuing for", string.match(key, "^(.+)%-") or key) + local items = nil + local count = 0 + for item, _ in pairs(data) do + print("found item", item) + if items == nil then + items = item + else + items = items .. "\0" .. item + end + count = count + 1 + if count == 1000 then + submit_backfeed(items, key) + items = nil + count = 0 + end + end + submit_backfeed(items, key) + end +end + +wget.callbacks.before_exit = function(exit_status, exit_status_string) + if killgrab then + return wget.exits.IO_FAIL + end + if abortgrab then + abort_item() + end + return exit_status +end diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..c3caaed --- /dev/null +++ b/pipeline.py @@ -0,0 +1,374 @@ +# encoding=utf8 +from distutils.version import StrictVersion +import datetime +import hashlib +import os +import re +import shutil +import socket +import sys +import time + +if sys.version_info[0] < 3: + from urllib import unquote +else: + from urllib.parse import unquote + +import seesaw +from seesaw.config import realize, NumberConfigValue +from seesaw.externalprocess import WgetDownload +from seesaw.item import ItemInterpolation, ItemValue +from seesaw.pipeline import Pipeline +from seesaw.project import Project +from seesaw.task import SimpleTask, LimitConcurrent +from seesaw.tracker import GetItemFromTracker, PrepareStatsForTracker, \ + UploadWithTracker, SendDoneToTracker +from seesaw.util import find_executable + +if StrictVersion(seesaw.__version__) < StrictVersion('0.8.5'): + raise Exception('This pipeline needs seesaw version 0.8.5 or higher.') + + +########################################################################### +# Find a useful Wget+Lua executable. +# +# WGET_AT will be set to the first path that +# 1. does not crash with --version, and +# 2. prints the required version string + +class HigherVersion: + def __init__(self, expression, min_version): + self._expression = re.compile(expression) + self._min_version = min_version + + def search(self, text): + for result in self._expression.findall(text): + if result >= self._min_version: + print('Found version {}.'.format(result)) + return True + + +WGET_AT = find_executable( + 'Wget+AT', + HigherVersion( + r'(GNU Wget 1\.[0-9]{2}\.[0-9]{1}-at\.[0-9]{8}\.[0-9]{2})[^0-9a-zA-Z\.-_]', + 'GNU Wget 1.21.3-at.20241119.01' + ), + [ + './wget-at', + '/home/warrior/data/wget-at-nss' + ] +) + +if not WGET_AT: + raise Exception('No usable Wget+At found.') + + +########################################################################### +# The version number of this pipeline definition. +# +# Update this each time you make a non-cosmetic change. +# It will be added to the WARC files and reported to the tracker. +VERSION = '20260629.01' +TRACKER_ID = 'blice' +TRACKER_HOST = 'legacy-api.arpa.li' +MULTI_ITEM_SIZE = 100 + + +########################################################################### +# This section defines project-specific tasks. +# +# Simple tasks (tasks that do not need any concurrency) are based on the +# SimpleTask class and have a process(item) method that is called for +# each item. +class CheckIP(SimpleTask): + def __init__(self): + SimpleTask.__init__(self, 'CheckIP') + self._counter = 0 + + def process(self, item): + # NEW for 2014! Check if we are behind firewall/proxy + + if self._counter <= 0: + item.log_output('Checking IP address.') + ip_set = set() + + ip_set.add(socket.gethostbyname('twitter.com')) + #ip_set.add(socket.gethostbyname('facebook.com')) + ip_set.add(socket.gethostbyname('youtube.com')) + ip_set.add(socket.gethostbyname('microsoft.com')) + ip_set.add(socket.gethostbyname('icanhas.cheezburger.com')) + ip_set.add(socket.gethostbyname('archiveteam.org')) + + if len(ip_set) != 5: + item.log_output('Got IP addresses: {0}'.format(ip_set)) + item.log_output( + 'Are you behind a firewall/proxy? That is a big no-no!') + raise Exception( + 'Are you behind a firewall/proxy? That is a big no-no!') + + # Check only occasionally + if self._counter <= 0: + self._counter = 10 + else: + self._counter -= 1 + + +class PrepareDirectories(SimpleTask): + def __init__(self, warc_prefix): + SimpleTask.__init__(self, 'PrepareDirectories') + self.warc_prefix = warc_prefix + + def process(self, item): + item_name = item['item_name'] + item_name_hash = hashlib.sha1(item_name.encode('utf8')).hexdigest() + escaped_item_name = item_name_hash + dirname = '/'.join((item['data_dir'], escaped_item_name)) + + if os.path.isdir(dirname): + shutil.rmtree(dirname) + + os.makedirs(dirname) + + item['item_dir'] = dirname + item['warc_file_base'] = '-'.join([ + self.warc_prefix, + item_name_hash, + time.strftime('%Y%m%d-%H%M%S') + ]) + + open('%(item_dir)s/%(warc_file_base)s.warc.gz' % item, 'w').close() + open('%(item_dir)s/%(warc_file_base)s_data.txt' % item, 'w').close() + + +class MoveFiles(SimpleTask): + def __init__(self): + SimpleTask.__init__(self, 'MoveFiles') + + def process(self, item): + os.rename('%(item_dir)s/%(warc_file_base)s.warc.gz' % item, + '%(data_dir)s/%(warc_file_base)s.warc.gz' % item) + os.rename('%(item_dir)s/%(warc_file_base)s_data.txt' % item, + '%(data_dir)s/%(warc_file_base)s_data.txt' % item) + + shutil.rmtree('%(item_dir)s' % item) + + +def normalize_string(s): + while True: + temp = unquote(s).strip().lower() + if temp == s: + break + s = temp + return s + + +class SetBadUrls(SimpleTask): + def __init__(self): + SimpleTask.__init__(self, 'SetBadUrls') + + def process(self, item): + item['item_name_original'] = item['item_name'] + items = item['item_name'].split('\0') + items_lower = [normalize_string(s) for s in items] + with open('%(item_dir)s/%(warc_file_base)s_bad-items.txt' % item, 'r') as f: + for s in { + normalize_string(s) for s in f + }: + index = items_lower.index(s) + item.log_output('Item {} is aborted.'.format(s)) + items.pop(index) + items_lower.pop(index) + item['item_name'] = '\0'.join(items) + + +class MaybeUploadWithTracker(UploadWithTracker): + def enqueue(self, item): + if len(item['item_name']) == 0: + item.log_output('Skipping UploadWithTracker.') + return self.complete_item(item) + return super(UploadWithTracker, self).enqueue(item) + + +class MaybeSendDoneToTracker(SendDoneToTracker): + def enqueue(self, item): + if len(item['item_name']) == 0: + return self.complete_item(item) + return super(MaybeSendDoneToTracker, self).enqueue(item) + + +def get_hash(filename): + with open(filename, 'rb') as in_file: + return hashlib.sha1(in_file.read()).hexdigest() + +CWD = os.getcwd() +PIPELINE_SHA1 = get_hash(os.path.join(CWD, 'pipeline.py')) +LUA_SHA1 = get_hash(os.path.join(CWD, 'blice.lua')) + +def stats_id_function(item): + d = { + 'pipeline_hash': PIPELINE_SHA1, + 'lua_hash': LUA_SHA1, + 'python_version': sys.version, + } + + return d + + +class WgetArgs(object): + def realize(self, item): + wget_args = [ + WGET_AT, + '-nv', + '--no-cookies', + '--host-lookups', 'dns', + '--hosts-file', '/dev/null', + '--resolvconf-file', '/dev/null', + '--dns-servers', '9.9.9.10,149.112.112.10,2620:fe::10,2620:fe::fe:10', + '--reject-reserved-subnets', + #'--prefer-family', ('IPv4' if 'PREFER_IPV4' in os.environ else 'IPv6'), + '--content-on-error', + '--lua-script', 'blice.lua', + '-o', ItemInterpolation('%(item_dir)s/wget.log'), + '--output-document', ItemInterpolation('%(item_dir)s/wget.tmp'), + '--truncate-output', + '-e', 'robots=off', + '--recursive', '--level=inf', + '--no-parent', + '--page-requisites', + '--timeout', '30', + '--connect-timeout', '1', + '--tries', 'inf', + '--domains', ','.join([ + 'blice.co.kr', + 'www.blice.co.kr', + 'cds.blice.co.kr' + ]), + '--span-hosts', + '--waitretry', '30', + '--warc-file', ItemInterpolation('%(item_dir)s/%(warc_file_base)s'), + '--warc-header', 'operator: Archive Team', + '--warc-header', 'x-wget-at-project-version: ' + VERSION, + '--warc-header', 'x-wget-at-project-name: ' + TRACKER_ID, + '--warc-dedup-url-agnostic', + '--impersonate', 'firefox148-h1', + '--header', 'Accept-Encoding: identity' + ] + + if '--concurrent' in sys.argv: + concurrency = int(sys.argv[sys.argv.index('--concurrent')+1]) + else: + concurrency = os.getenv('CONCURRENT_ITEMS') + if concurrency is None: + concurrency = 2 + item['concurrency'] = str(concurrency) + + for item_name in item['item_name'].split('\0'): + item_type, item_value = item_name.split(':', 1) + wget_args.extend(['--warc-header', 'x-wget-at-project-item-name: '+item_name]) + if item_type == 'novel': + wget_args.extend(['--warc-header', 'blice-novel: '+item_value]) + wget_args.append('https://blice.co.kr/web/detail.kt?novelId={}'.format(item_value)) + elif item_type == 'times': + wget_args.extend(['--warc-header', 'blice-times: '+item_value]) + wget_args.append('https://blice.co.kr/web/viewer.kt?timesId={}'.format(item_value)) + elif item_type == 'event': + wget_args.extend(['--warc-header', 'blice-event: '+item_value]) + wget_args.append('https://blice.co.kr/event/eventPage.kt?eventSeq={}'.format(item_value)) + elif item_type == 'document': + wget_args.extend(['--warc-header', 'blice-document: '+item_value]) + wget_args.append('https://blice.co.kr/web/documents/detail.kt?documentSeq={}'.format(item_value)) + elif item_type == 'notice': + wget_args.extend(['--warc-header', 'blice-notice: '+item_value]) + wget_args.append('https://blice.co.kr/web/support/noticeDetail.kt?noticeSeq={}'.format(item_value)) + elif item_type == 'user': + wget_args.extend(['--warc-header', 'blice-user: '+item_value]) + wget_args.append('https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq={}'.format(item_value)) + elif item_type == 'asset': + url = 'https://' + item_value + wget_args.extend(['--warc-header', 'blice-asset: '+url]) + wget_args.append(url) + else: + raise Exception('Unknown item') + + item['item_name_newline'] = item['item_name'].replace('\0', '\n') + + if 'bind_address' in globals(): + wget_args.extend(['--bind-address', globals()['bind_address']]) + print('') + print('*** Wget will bind address at {0} ***'.format( + globals()['bind_address'])) + print('') + + return realize(wget_args, item) + + +########################################################################### +# Initialize the project. +# +# This will be shown in the warrior management panel. The logo should not +# be too big. The deadline is optional. +project = Project( + title=TRACKER_ID, + project_html=''' + +

Blice Website · Leaderboard · Wiki

+

Archiving Blice.

+ ''', + utc_deadline=datetime.datetime(2026, 6, 30, 15, 0, 0) +) + + +pipeline = Pipeline( + CheckIP(), + GetItemFromTracker('https://{}/{}/multi={}/' + .format(TRACKER_HOST, TRACKER_ID, MULTI_ITEM_SIZE), + downloader, VERSION), + PrepareDirectories(warc_prefix=TRACKER_ID), + WgetDownload( + WgetArgs(), + max_tries=1, + accept_on_exit_code=[0, 4, 8], + env={ + 'item_dir': ItemValue('item_dir'), + 'item_names': ItemValue('item_name_newline'), + 'warc_file_base': ItemValue('warc_file_base'), + 'concurrency': ItemValue('concurrency') + } + ), + SetBadUrls(), + PrepareStatsForTracker( + defaults={'downloader': downloader, 'version': VERSION}, + file_groups={ + 'data': [ + ItemInterpolation('%(item_dir)s/%(warc_file_base)s.warc.gz') + ] + }, + id_function=stats_id_function, + ), + MoveFiles(), + LimitConcurrent(NumberConfigValue(min=1, max=20, default='20', + name='shared:rsync_threads', title='Rsync threads', + description='The maximum number of concurrent uploads.'), + MaybeUploadWithTracker( + 'https://%s/%s' % (TRACKER_HOST, TRACKER_ID), + downloader=downloader, + version=VERSION, + files=[ + ItemInterpolation('%(data_dir)s/%(warc_file_base)s.warc.gz'), + ItemInterpolation('%(data_dir)s/%(warc_file_base)s_data.txt') + ], + rsync_target_source_path=ItemInterpolation('%(data_dir)s/'), + rsync_extra_args=[ + '--recursive', + '--min-size', '1', + '--no-compress', + '--compress-level', '0' + ] + ), + ), + MaybeSendDoneToTracker( + tracker_url='https://%s/%s' % (TRACKER_HOST, TRACKER_ID), + stats=ItemValue('stats') + ) +)