local urlparse = require("socket.url")
local http = require("socket.http")
local cjson = require("cjson")
local utf8 = require("utf8")

local item_dir = os.getenv("item_dir")
local warc_file_base = os.getenv("warc_file_base")
local concurrency = tonumber(os.getenv("concurrency"))
local item_type = nil
local item_name = nil
local item_value = nil
local ids = {}

local url_count = 0
local tries = 0
local downloaded = {}
local seen_200 = {}
local addedtolist = {}
local abortgrab = false
local killgrab = false
local logged_response = false

local discovered_items = {}
local bad_items = {}

local retry_url = false

local item_patterns = {
  ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/viewer%.kt%?.*timesId=([0-9]+)"]="times",
  ["^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?.*timesId=([0-9]+)"]="times",
  ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/detail%.kt%?.*novelId=([0-9]+)"]="novel",
  ["^https?://[^/]*blice%.co%.kr/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event",
  ["^https?://[^/]*blice%.co%.kr/mw/event/eventPage%.kt%?.*eventSeq=([0-9]+)"]="event",
  ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/documents/detail%.kt%?.*documentSeq=([0-9]+)"]="document",
  ["^https?://[^/]*blice%.co%.kr/web/support/noticeDetail%.kt%?.*noticeSeq=([0-9]+)"]="notice",
  ["^https?://[^/]*blice%.co%.kr/[mw][we]?b?/retrieveSpecificUserProfile%.kt%?.*userNicknameSeq=([0-9]+)"]="user",
  ["^https?://(cds%.blice%.co%.kr/download%?file=.+)"]="asset"
}

abort_item = function(item)
  abortgrab = true
  if not item then
    item = item_name
  end
  if not bad_items[item] then
    io.stdout:write("Aborting item " .. item .. ".\n")
    io.stdout:flush()
    bad_items[item] = true
  end
end

kill_grab = function(item)
  io.stdout:write("Aborting crawling.\n")
  io.stdout:flush()
  killgrab = true
end

read_file = function(file)
  local f = assert(io.open(file, "rb"))
  local data = f:read("*all")
  f:close()
  return data
end

processed = function(url)
  if downloaded[url] or addedtolist[url] then
    return true
  end
  return false
end

discover_item = function(target, item)
  if item ~= item_name and not target[item] then
--print("discovered", item)
    target[item] = true
    return true
  end
  return false
end

query_param = function(url, name)
  local query = string.match(url, "%?(.+)$")
  if not query then
    return nil
  end
  for key, value in string.gmatch(query, "([^&=]+)=?([^&]*)") do
    if key == name then
      return urlparse.unescape(value)
    end
  end
  return nil
end

find_item = function(url)
  url = urlparse.unescape(url)
  for pattern, type_ in pairs(item_patterns) do
    local value = string.match(url, pattern)
    if value then
      return {
        ["type"]=type_,
        ["value"]=value
      }
    end
  end
end

set_item = function(url)
  local found = find_item(url)
  if found then
    local new_item_type = found["type"]
    local new_item_value = found["value"]
    local new_item_name = new_item_type .. ":" .. new_item_value
    if new_item_name ~= item_name then
      item_value = new_item_value
      item_type = new_item_type
      ids = {}
      ids[item_value] = true
      abortgrab = false
      tries = 0
      retry_url = false
      item_name = new_item_name
      print("Archiving item " .. item_name)
    end
  end
end

allowed = function(url, body_data)
  if not string.match(url, "^https?://") then
    return false
  end

  if not (
    string.match(url, "^https?://www%.blice%.co%.kr/")
    or string.match(url, "^https?://blice%.co%.kr/")
    or string.match(url, "^https?://cds%.blice%.co%.kr/")
  ) then
    return false
  end

  local found = find_item(url)
  if found then
    local found_item_name = found["type"] .. ":" .. found["value"]
    if found_item_name ~= item_name then
      discover_item(discovered_items, found_item_name)
      return false
    end
    return true
  end

  for _, pattern in ipairs({
    "^https?://[^/]*blice%.co%.kr/[mw][we]?b?/comment%.kt%?",
    "^https?://[^/]*blice%.co%.kr/[mw][we]?b?/other_more_list%.kt%?",
    "^https?://[^/]*blice%.co%.kr/web/readMoreTimesList_ajax%?",
    "^https?://[^/]*blice%.co%.kr/mw/readMoreTimesList_ajax%?",
    "^https?://[^/]*blice%.co%.kr/web/viewer_readMoreTimesList_ajax%?",
    "^https?://[^/]*blice%.co%.kr/web/commentList_ajax%?",
    "^https?://[^/]*blice%.co%.kr/web/event/commentList_ajax%?",
    "^https?://[^/]*blice%.co%.kr/[mw][we]?b?/commentListDetail%.kt%?"
  }) do
    if string.match(url, pattern) then
      for _, name in ipairs({
        "novelId",
        "novel_no",
        "novelSeq",
        "timesId",
        "novelTimesSeq",
        "exceptionalEventSeq",
        "eventSeq"
      }) do
        if ids[query_param(url, name)] then
          return true
        end
      end
    end
  end

  if body_data then
    return true
  end

  if item_type == "event"
    and (
      string.match(url, "^https?://[^/]*blice%.co%.kr/event/more%.kt%?")
      or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?")
    ) then
    return true
  end

  return false
end

wget.callbacks.download_child_p = function(urlpos, parent, depth, start_url_parsed, iri, verdict, reason)
  return false
end

decode_codepoint = function(newurl)
  newurl = string.gsub(
    newurl, "\\[uU]([0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F])",
    function (s)
      return utf8.char(tonumber(s, 16))
    end
  )
  return newurl
end

wget.callbacks.get_urls = function(file, url, is_css, iri)
  local urls = {}

  downloaded[url] = true
  set_item(url)

  if abortgrab then
    return {}
  end

  local function fix_case(newurl)
    if not string.match(newurl, "^https?://[^/]") then
      return newurl
    end
    if string.match(newurl, "^https?://[^/]+$") then
      newurl = newurl .. "/"
    end
    local a, b = string.match(newurl, "^(https?://[^/]+/)(.*)$")
    return string.lower(a) .. b
  end

  local function check(newurl, headers, body_data)
    if not string.match(newurl, "^https?://") then
      return nil
    end
    newurl = decode_codepoint(newurl)
    if string.match(newurl, "[%s\\]") then
      return nil
    end
    newurl = fix_case(newurl)
    if not body_data
      and string.match(newurl, "^https?://www%.blice%.co%.kr/")
      and not string.match(newurl, "_ajax%?")
      and not string.match(newurl, "/api/") then
      newurl = string.gsub(newurl, "^https?://www%.blice%.co%.kr/", "https://blice.co.kr/")
    end
    local url = string.match(newurl, "^([^#]+)")
    local url_ = url
    while string.match(url_, "&") do
      url_ = string.gsub(url_, "&", "&")
    end
    local method = body_data and "POST" or "GET"
    local key = method .. "\0" .. url_ .. "\0" .. tostring(body_data)
    if not processed(key)
      and (body_data or not processed(url_))
      and allowed(url_, body_data) then
      local url_data = {
        url=url_,
        headers=headers or {}
      }
      if body_data then
        url_data["body_data"] = body_data
        url_data["method"] = method
      end
      table.insert(urls, url_data)
      addedtolist[key] = true
      if not body_data then
        addedtolist[url_] = true
        addedtolist[url] = true
      end
      return true
    end
  end

  local function checknewurl(newurl)
    newurl = decode_codepoint(newurl)
    if string.match(newurl, "['\"><]") then
      return nil
    end
    if string.match(newurl, "^https?:////") then
      check(string.gsub(newurl, ":////", "://"))
    elseif string.match(newurl, "^https?://") then
      check(newurl)
    elseif string.match(newurl, "^https?:\\/\\?/") then
      check(string.gsub(newurl, "\\", ""))
    elseif string.match(newurl, "^\\/\\/") then
      checknewurl(string.gsub(newurl, "\\", ""))
    elseif string.match(newurl, "^//") then
      check(urlparse.absolute(url, newurl))
    elseif string.match(newurl, "^\\/") then
      checknewurl(string.gsub(newurl, "\\", ""))
    elseif string.match(newurl, "^/") then
      check(urlparse.absolute(url, newurl))
    elseif string.match(newurl, "^%.%./") then
      if string.match(url, "^https?://[^/]+/[^/]+/") then
        check(urlparse.absolute(url, newurl))
      else
        checknewurl(string.match(newurl, "^%.%.(/.+)$"))
      end
    elseif string.match(newurl, "^%./") then
      check(urlparse.absolute(url, newurl))
    end
  end

  local function checknewshorturl(newurl)
    newurl = decode_codepoint(newurl)
    if string.match(newurl, "^%?") then
      check(urlparse.absolute(url, newurl))
    elseif not (
      string.match(newurl, "^https?:\\?/\\?//?/?")
      or string.match(newurl, "^[/\\]")
      or string.match(newurl, "^%./")
      or string.match(newurl, "^[jJ]ava[sS]cript:")
      or string.match(newurl, "^[mM]ail[tT]o:")
      or string.match(newurl, "^data:")
      or string.match(newurl, "^%${")
    ) then
      check(urlparse.absolute(url, newurl))
    end
  end

  local function check_media(data)
    local values = {data}
    if type(data) == "table" then
      for _, key in ipairs({
        "epubCoverUrl",
        "epubUrl",
        "filePath1",
        "filePath2",
        "imagePath1",
        "imagePath2",
        "profileImgPath",
        "sticonUrl",
        "timesCoverImagePath",
        "timesFilePath",
        "userImage",
        "workImageWideUrl",
        "workThumbUrl"
      }) do
        if data[key] then
          table.insert(values, data[key])
        end
      end
      if type(data["webtoonImages"]) == "table" then
        for _, value in ipairs(data["webtoonImages"]) do
          table.insert(values, value)
        end
      end
      for _, pair in ipairs({
        {"epubSavePath", "epubNewName"},
        {"epubCoverSavePath", "epubCoverNewName"}
      }) do
        local save_path = data[pair[1]]
        local new_name = data[pair[2]]
        if type(save_path) == "string" and type(new_name) == "string" then
          save_path = string.gsub(save_path, "^/nas/noveltimes/epub/", "/epub/noveltimes/")
          if string.match(save_path, "^/epub/noveltimes/") then
            table.insert(values, save_path .. new_name)
          end
        end
      end
    end
    for _, value in ipairs(values) do
      if type(value) == "string" then
        if string.match(value, "^https?://") then
          check(value)
        elseif string.match(value, "^/") then
          check("https://cds.blice.co.kr/download?file=" .. value)
        end
      end
    end
  end

  local function check_xmlhttprequest(url, referer, body)
    local headers = {
      ["Content-Type"]="application/json; charset=UTF-8",
      ["X-Requested-With"]="XMLHttpRequest",
      ["Referer"]=referer
    }
    check(string.match(url, "^([^?]+)"), headers, body)
    check(url, headers, body)
  end

  local function check_web_mw(path_query, web_extra, mw_extra)
    web_extra = web_extra or ""
    if not mw_extra then
      mw_extra = web_extra
    end
    for _, data in ipairs({
      {"web", web_extra},
      {"mw", mw_extra}
    }) do
      check("https://blice.co.kr/" .. data[1] .. "/" .. path_query .. data[2])
    end
  end

  local function check_readmore(endpoint, page_no, order_by, novel_type_seq)
    local path = "readMoreTimesList_ajax"
    local body_novel_no = item_value
    if endpoint == "viewer" then
      endpoint = "web"
      path = "viewer_readMoreTimesList_ajax"
      order_by = "recent"
      body_novel_no = '"' .. item_value .. '"'
    end
    local url = "https://blice.co.kr/" .. endpoint .. "/" .. path
      .. "?novel_no=" .. item_value
      .. "&orderBy=" .. order_by
      .. "&pageNo=" .. tostring(page_no)
    local body = '{"novel_no":' .. body_novel_no
    if novel_type_seq then
      url = url .. "&novelTypeSeq=" .. novel_type_seq
      body = body .. ',"novelTypeSeq":' .. novel_type_seq
    end
    body = body
      .. ',"orderBy":"' .. order_by .. '"'
      .. ',"pageNo":' .. tostring(page_no)
    if endpoint == "mw" or path == "viewer_readMoreTimesList_ajax" then
      body = body .. ',"row":20'
      url = url .. "&row=20"
    end
    body = body .. "}"
    check_xmlhttprequest(
      url,
      "https://blice.co.kr/" .. endpoint .. "/detail.kt?novelId=" .. item_value,
      body
    )
  end

  local function check_comment_ajax(novel_seq, times_seq, comment_type, page_no)
    local referer = "https://blice.co.kr/web/comment.kt?novelId=" .. novel_seq
      .. "&commentType=" .. comment_type
    if times_seq ~= "0" then
      referer = "https://blice.co.kr/web/comment.kt?timesId=" .. times_seq
        .. "&novelId=" .. novel_seq
        .. "&commentType=" .. comment_type
    end
    local body = '{"novelSeq":' .. novel_seq
      .. ',"novelTimesSeq":' .. times_seq
      .. ',"commentType":"' .. comment_type .. '"'
      .. ',"orderBy":"recent"'
      .. ',"pageNo":' .. tostring(page_no)
      .. "}"
    check_xmlhttprequest(
      "https://blice.co.kr/web/commentList_ajax?novelSeq=" .. novel_seq
        .. "&novelTimesSeq=" .. times_seq
        .. "&commentType=" .. comment_type
        .. "&orderBy=recent"
        .. "&pageNo=" .. tostring(page_no),
      referer,
      body
    )
  end

  if status_code == 200
    and allowed(url)
    and (
      string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+%.kt")
      or string.match(url, "^https?://[^/]*blice%.co%.kr/[^?]+_ajax%?")
      or string.match(url, "^https?://[^/]*blice%.co%.kr/api/novel/v1/viewerAjax%?")
      or string.match(url, "^https?://[^/]*blice%.co%.kr/event/card%?")
    ) then
    local html = read_file(file)
    if string.match(url, "^https?://blice%.co%.kr/web/detail%.kt%?")
      and query_param(url, "novelId") == item_value then
      check_web_mw(
        "detail.kt?novelId=" .. item_value,
        "&isPrologTimes=false&voucherXCnt=0&voucherYCnt=0&voucherY1Cnt=0&voucherY3Cnt=0&voucherLCnt=0&voucherSCnt=0"
          .. "&viewVoucherRentCnt=0&viewVoucherHaveCnt=0&needGaLog=true&isLogin=false"
          .. "&prevUrl=http%3A%2F%2Fblice.co.kr%2Fweb%2Fviewer.kt",
        ""
      )
      check_web_mw("comment.kt?novelId=" .. item_value .. "&commentType=novel", "", "&quickViewYn=N")
      for _, data in ipairs({
        {"web", "seq"},
        {"web", "recent"},
        {"web", "regDate"},
        {"mw", "seq"},
        {"mw", "recent"}
      }) do
        check_readmore(data[1], 1, data[2])
      end
      local novel_type_seq = string.match(html, "novelTypeSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)")
        or string.match(html, "novelTypeSeq=([0-9]+)")
      if novel_type_seq then
        for _, order_by in ipairs({"recent", "regDate"}) do
          check_readmore("web", 1, order_by, novel_type_seq)
        end
      end
      check_readmore("viewer", 1)
      check_comment_ajax(item_value, "0", "novel", 1)
    elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?")
      and query_param(url, "timesId") == item_value then
      local times_type = query_param(url, "timesType")
      if times_type == "" then
        times_type = nil
      end
      times_type = times_type
        or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)")
        or string.match(html, "timesType=([a-zA-Z]+)")
      if not times_type
        or not string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)") then
        return urls
      end
      local viewer_path_query = "viewer.kt?timesId=" .. item_value .. "&timesType=" .. times_type
      check_web_mw(viewer_path_query, "&quickViewYn=N")
      check_web_mw(viewer_path_query, "&quickViewYn=")
      check_web_mw(viewer_path_query)
      check("https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value)
      check_xmlhttprequest(
        "https://blice.co.kr/api/novel/v1/viewerAjax?timesId=" .. item_value .. "&timesType=" .. times_type .. "&quickViewYn=N",
        "https://blice.co.kr/mw/viewer.kt?timesId=" .. item_value .. "&timesType=" .. times_type .. "&quickViewYn=N",
        '{"timesId":' .. item_value .. ',"timesType":"' .. times_type .. '","quickViewYn":"N"}'
      )
    elseif string.match(url, "^https?://blice%.co%.kr/event/eventPage%.kt%?")
      and query_param(url, "eventSeq") == item_value then
      check("https://blice.co.kr/mw/event/eventPage.kt?eventSeq=" .. item_value)
      check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1")
      check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. item_value .. "&pageNo=1&exceptionalEventSeq=" .. item_value .. "&pageNo=1")
    elseif string.match(url, "^https?://blice%.co%.kr/web/documents/detail%.kt%?")
      and query_param(url, "documentSeq") == item_value then
      check_web_mw("documents/detail.kt?documentSeq=" .. item_value, "&start=1", "")
    elseif string.match(url, "^https?://blice%.co%.kr/mw/retrieveSpecificUserProfile%.kt%?")
      and query_param(url, "userNicknameSeq") == item_value then
      if query_param(url, "userType") == "AUTHOR_PEN" then
        local penname_id = query_param(url, "userPennameSeq")
        check_xmlhttprequest(
          "https://blice.co.kr/web/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userPennameSeq=" .. penname_id .. "&userNicknameSeq=" .. item_value,
          "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. penname_id,
          '{"userType":"AUTHOR_PEN","userPennameSeq":"' .. penname_id .. '","userNicknameSeq":"' .. item_value .. '","adultViewYn":1,"onlyPenYn":"Y"}'
        )
      else
        for _, user_type in ipairs({
          "USER",
          "AUTHOR_NICK"
        }) do
          check_xmlhttprequest(
            "https://blice.co.kr/web/retrieveSpecificUserProfile.kt?userType=" .. user_type .. "&userNicknameSeq=" .. item_value,
            "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item_value,
            '{"userType":"' .. user_type .. '","userNicknameSeq":"' .. item_value .. '"}'
          )
        end
      end
    end
    if string.match(url, "/readMoreTimesList_ajax%?")
      or string.match(url, "/viewer_readMoreTimesList_ajax%?") then
      local response = cjson.decode(html)["response"]
      local count = 0
      local times_list = response["timesResList"]
      if string.match(url, "/mw/")
        or string.match(url, "/viewer_readMoreTimesList_ajax%?") then
        times_list = response
      end
      for _, item in ipairs(times_list) do
        count = count + 1
        check("https://blice.co.kr/web/viewer.kt?timesId=" .. item["novelTimesSeq"])
        check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
        check_media(item)
      end
      local page_no = tonumber(query_param(url, "pageNo"))
      local order_by = query_param(url, "orderBy")
      local novel_type_seq = query_param(url, "novelTypeSeq")
      local is_mw = string.match(url, "/mw/")
      local is_viewer = string.match(url, "/viewer_readMoreTimesList_ajax%?")
      local readmore_endpoint = "web"
      if is_mw then
        readmore_endpoint = "mw"
      elseif is_viewer then
        readmore_endpoint = "viewer"
      end
      if not is_mw and not is_viewer then
        for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do
          new_page = tonumber(new_page)
          if new_page > page_no then
            check_readmore("web", new_page, order_by, novel_type_seq)
          end
        end
      end
      if count > 0 then
        if readmore_endpoint == "mw" then
          check_readmore(readmore_endpoint, page_no + 1, order_by)
        elseif readmore_endpoint == "viewer" then
          check_readmore(readmore_endpoint, page_no + 1)
        else
          check_readmore(readmore_endpoint, page_no + 1, order_by, novel_type_seq)
        end
      end
    elseif string.match(url, "/event/commentList_ajax%?") then
      local response = cjson.decode(html)["response"]
      local count = 0
      for _, item in ipairs(response["eventCommentResList"]) do
        count = count + 1
        check_media(item)
      end
      local page_no = tonumber(query_param(url, "pageNo"))
      local event_seq = query_param(url, "exceptionalEventSeq")
      for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)") do
        new_page = tonumber(new_page)
        if new_page > page_no then
          check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(new_page))
        end
      end
      if count >= 30 then
        check("https://blice.co.kr/web/event/commentList_ajax?exceptionalEventSeq=" .. event_seq .. "&pageNo=" .. tostring(page_no + 1))
      end
    elseif string.match(url, "/event/card%?") then
      local count = 0
      for _, item in ipairs(cjson.decode(html)["response"]["workList"]) do
        count = count + 1
        check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
        check_media(item)
      end
      if count > 0 then
        check("https://blice.co.kr/event/card?cardListSeq=" .. query_param(url, "cardListSeq")
          .. "&startNo=" .. tostring(tonumber(query_param(url, "startNo")) + 1))
      end
    elseif string.match(url, "/commentList_ajax%?") then
      local response = cjson.decode(html)["response"]
      local count = 0
      local comment_type = query_param(url, "commentType")
      local times_seq = query_param(url, "novelTimesSeq")
      for _, item in ipairs(response["timesCommentResList"]) do
        count = count + 1
        check_media(item)
        check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=" .. item["userNicknameSeq"])
        check_web_mw(
          "commentListDetail.kt?novelId=" .. item["novelSeq"] .. "&timesId=" .. item["novelTimesSeq"],
          "&commentId=" .. item["timesCommentSeq"] .. "&commentType=" .. comment_type,
          "&timesType=" .. comment_type
            .. "&commentId=" .. item["timesCommentSeq"]
            .. "&commentType=" .. comment_type
            .. "&quickViewYn=N"
        )
      end
      local page_no = tonumber(query_param(url, "pageNo"))
      for new_page in string.gmatch(response["pageInfo"], "fncGetList%(([0-9]+)%)") do
        new_page = tonumber(new_page)
        if new_page > page_no then
          check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, new_page)
        end
      end
      if count >= 30 then
        check_comment_ajax(query_param(url, "novelSeq"), times_seq, comment_type, page_no + 1)
      end
    elseif string.match(url, "^https?://[^/]*blice%.co%.kr/web/retrieveSpecificUserProfile%.kt%?") then
      local response = cjson.decode(html)
      check_media(response["specificUserProfile"])
      if query_param(url, "userType") ~= "USER" then
        for _, item in ipairs(response["authorNovelList"]) do
          check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y"
            .. "&userType=AUTHOR_PEN&userNicknameSeq=" .. item_value .. "&userPennameSeq=" .. item["authorPenSeq"])
          check("https://blice.co.kr/web/detail.kt?novelId=" .. item["novelSeq"])
          check_media(item)
        end
      else
        for _, item in ipairs(response["userDocumentsList"]) do
          check("https://blice.co.kr/web/documents/detail.kt?documentSeq=" .. item["documentSeq"])
        end
      end
    elseif string.match(url, "/api/novel/v1/viewerAjax%?") then
      local response = cjson.decode(html)["response"]
      check("https://blice.co.kr/web/detail.kt?novelId=" .. response["novelSeq"])
      check_media(response)
      for _, key in ipairs({"prev", "next"}) do
        local adjacent = response[key]
        if type(adjacent) == "table" then
          check("https://blice.co.kr/web/viewer.kt?timesId=" .. adjacent["novelTimesSeq"])
        end
      end
    else
      for _, data in ipairs({
        {"novelId=([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"},
        {"data%-novelSeq=['\"]?([0-9]+)", "https://blice.co.kr/web/detail.kt?novelId=%s"},
        {"data%-first%-timesId=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"},
        {"data%-times%-seq=\"([0-9]+)\"", "https://blice.co.kr/web/viewer.kt?timesId=%s"},
        {"data%-nickNameSeq=\"([0-9]+)\"", "https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=N&userType=USER&userNicknameSeq=%s"},
        {"eventSeq=([0-9]+)", "https://blice.co.kr/event/eventPage.kt?eventSeq=%s"},
        {"documentSeq=([0-9]+)", "https://blice.co.kr/web/documents/detail.kt?documentSeq=%s"},
        {"noticeSeq=([0-9]+)", "https://blice.co.kr/web/support/noticeDetail.kt?noticeSeq=%s"}
      }) do
        for value in string.gmatch(html, data[1]) do
          check(string.format(data[2], value))
        end
      end
      for times_id in string.gmatch(html, "timesId=([0-9]+)") do
        check("https://blice.co.kr/web/viewer.kt?timesId=" .. times_id)
      end
      if query_param(url, "novelId") == item_value
        or query_param(url, "novel_no") == item_value
        or query_param(url, "novelSeq") == item_value then
        for penname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\"") do
          check_web_mw("other_more_list.kt?novelId=" .. item_value .. "&pennameId=" .. penname_id)
        end
        for penname_id, nickname_id in string.gmatch(html, "data%-penname%-seq=\"([0-9]+)\" data%-nickname%-seq=\"([0-9]+)\" data%-adult%-view%-yn=\"[^\"]*\"") do
          check("https://blice.co.kr/mw/retrieveSpecificUserProfile.kt?onlyPenYn=Y"
            .. "&userType=AUTHOR_PEN&userNicknameSeq=" .. nickname_id .. "&userPennameSeq=" .. penname_id)
        end
      elseif string.match(url, "^https?://blice%.co%.kr/[mw][we]?b?/viewer%.kt%?") then
        local novel_no = string.match(html, "var%s+novelNo%s*=%s*['\"]?([0-9]+)")
        if not novel_no then
          return urls
        end
        check("https://blice.co.kr/web/detail.kt?novelId=" .. novel_no)
        local times_type = query_param(url, "timesType")
        if times_type == "" then
          times_type = nil
        end
        times_type = times_type
          or string.match(html, "timesType%s*=%s*['\"]?([a-zA-Z]+)")
          or string.match(html, "timesType=([a-zA-Z]+)")
        if not times_type then
          return urls
        end
        check_web_mw("comment.kt?timesId=" .. item_value .. "&novelId=" .. novel_no
          .. "&commentType=" .. times_type, "", "&quickViewYn=N")
        check("https://blice.co.kr/mw/comment.kt?timesId=" .. item_value
          .. "&novelId=" .. novel_no
          .. "&commentType=" .. times_type
          .. "&quickViewYn=N&hback=%2Fmw%2Fviewer.kt%3FtimesId%3D" .. item_value
          .. "%26timesType%3D" .. times_type
          .. "%26quickViewYn%3DN")
        check_comment_ajax(novel_no, item_value, times_type, 1)
      elseif string.match(url, "^https?://blice%.co%.kr/event/")
        or string.match(url, "^https?://blice%.co%.kr/mw/event/") then
        for card_list_id in string.gmatch(html, "cardListSeq[\"']?%s*[:=]%s*[\"']?([0-9]+)") do
          check("https://blice.co.kr/event/card?cardListSeq=" .. card_list_id .. "&startNo=2")
        end
      end
      for newurl in string.gmatch(string.gsub(html, "&[qQ][uU][oO][tT];", '"'), '([^"]+)') do
        checknewurl(newurl)
      end
      for newurl in string.gmatch(string.gsub(html, "&#039;", "'"), "([^']+)") do
        checknewurl(newurl)
      end
      for newurl in string.gmatch(html, "[^%-]href='([^']+)'") do
        checknewshorturl(newurl)
      end
      for newurl in string.gmatch(html, '[^%-]href="([^"]+)"') do
        checknewshorturl(newurl)
      end
      for newurl in string.gmatch(html, "[^%-]src='([^']+)'") do
        checknewshorturl(newurl)
      end
      for newurl in string.gmatch(html, '[^%-]src="([^"]+)"') do
        checknewshorturl(newurl)
      end
      for newurl in string.gmatch(html, ":%s*url%(([^%)]+)%)") do
        newurl = string.gsub(newurl, "^['\"]", "")
        newurl = string.gsub(newurl, "['\"]$", "")
        checknewurl(newurl)
      end
      html = string.gsub(html, "&gt;", ">")
      html = string.gsub(html, "&lt;", "<")
      for newurl in string.gmatch(html, ">%s*([^<%s]+)") do
        checknewurl(newurl)
      end
    end
  end

  return urls
end

wget.callbacks.write_to_warc = function(url, http_stat)
  status_code = http_stat["statcode"]
  set_item(url["url"])
  url_count = url_count + 1
  io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n")
  io.stdout:flush()
  logged_response = true
  if not item_name then
    error("No item name found.")
  end
  if http_stat["len"] == 0
    and status_code < 300 then
    retry_url = true
    return false
  end
  if status_code ~= 200
    and status_code ~= 302 then
    retry_url = true
    return false
  end
  if abortgrab then
    print("Not writing to WARC.")
    return false
  end
  retry_url = false
  tries = 0
  return true
end

wget.callbacks.httploop_result = function(url, err, http_stat)
  status_code = http_stat["statcode"]

  if not logged_response then
    url_count = url_count + 1
    io.stdout:write(url_count .. "=" .. status_code .. " " .. url["url"] .. " \n")
    io.stdout:flush()
  end
  logged_response = false

  if killgrab then
    return wget.actions.ABORT
  end

  set_item(url["url"])
  if not item_name then
    error("No item name found.")
  end

  if abortgrab then
    abort_item()
    return wget.actions.EXIT
  end

  if status_code == 0 or retry_url then
    io.stdout:write("Server returned bad response. ")
    io.stdout:flush()
    tries = tries + 1
    local maxtries = 6
    if tries > maxtries then
      io.stdout:write(" Skipping.\n")
      io.stdout:flush()
      tries = 0
      abort_item()
      return wget.actions.EXIT
    end
    local sleep_time = math.random(
      math.floor(math.pow(2, tries-0.5)),
      math.floor(math.pow(2, tries))
    )
    io.stdout:write("Sleeping " .. sleep_time .. " seconds.\n")
    io.stdout:flush()
    os.execute("sleep " .. sleep_time)
    return wget.actions.CONTINUE
  else
    if status_code == 200 or status_code == 206 then
      if not seen_200[url["url"]] then
        seen_200[url["url"]] = 0
      end
      seen_200[url["url"]] = seen_200[url["url"]] + 1
    end
    downloaded[url["url"]] = true
  end

  if status_code >= 300 and status_code <= 399 then
    local newloc = urlparse.absolute(url["url"], http_stat["newloc"])
    if processed(newloc) or not allowed(newloc) then
      tries = 0
      return wget.actions.EXIT
    end
  end

  tries = 0

  return wget.actions.NOTHING
end

wget.callbacks.finish = function(start_time, end_time, wall_time, numurls, total_downloaded_bytes, total_download_time)
  local function submit_backfeed(items, key)
    local tries = 0
    local maxtries = 5
    while tries < maxtries do
      if killgrab then
        return false
      end
      local body, code, headers, status = http.request(
        "https://legacy-api.arpa.li/backfeed/legacy/" .. key,
        items .. "\0"
      )
      if code == 200 and body ~= nil and cjson.decode(body)["status_code"] == 200 then
        io.stdout:write(string.match(body, "^(.-)%s*$") .. "\n")
        io.stdout:flush()
        return nil
      end
      io.stdout:write("Failed to submit discovered URLs." .. tostring(code) .. tostring(body) .. "\n")
      io.stdout:flush()
      os.execute("sleep " .. math.floor(math.pow(2, tries)))
      tries = tries + 1
    end
    kill_grab()
    error()
  end

  local file = io.open(item_dir .. "/" .. warc_file_base .. "_bad-items.txt", "w")
  for url, _ in pairs(bad_items) do
    file:write(url .. "\n")
  end
  file:close()
  for key, data in pairs({
    ["blice-tob75dmr5nkp494q"] = discovered_items
  }) do
    print("queuing for", string.match(key, "^(.+)%-") or key)
    local items = nil
    local count = 0
    for item, _ in pairs(data) do
      print("found item", item)
      if items == nil then
        items = item
      else
        items = items .. "\0" .. item
      end
      count = count + 1
      if count == 1000 then
        submit_backfeed(items, key)
        items = nil
        count = 0
      end
    end
    submit_backfeed(items, key)
  end
end

wget.callbacks.before_exit = function(exit_status, exit_status_string)
  if killgrab then
    return wget.exits.IO_FAIL
  end
  if abortgrab then
    abort_item()
  end
  return exit_status
end