Rspamd で ASCII Smuggling（不可視 Unicode タグ）を検出する

--
-- Unicode Tagsとゼロ幅スペースを検出 for Rspamd 4.1.x
--
-- 自動転送している場合は、除去オプションをtrueにすると転送先でのDKIM再検証に失敗する
--
--------------------------------------------------------
-- ユーザー設定
--------------------------------------------------------
-- Unicode Tags（U+E0000..U+E007F）を除去
local REMOVE_UNICODE_TAGS = false

-- ゼロ幅スペース（U+200B）を除去（検出・スコアの対象ではない）
local REMOVE_ZERO_WIDTH_SPACES = false

-- 除去を外部受信に限定（SMTP認証あり・ローカルIP・接続元不明は除外）
local CLEANUP_INBOUND_ONLY = true

-- DKIM/ARC署名保護を無視
local IGNORE_SIGNATURE_PROTECTION = false

-- MIME除去処理のサイズ上限
local MAX_SANITIZE_BYTES = 10 * 1024 * 1024

--------------------------------------------------------
local lua_mime = require "lua_mime"
local rspamd_util = require "rspamd_util"
local unpack_fn = table.unpack or unpack

local CLEANUP_ENABLED = REMOVE_UNICODE_TAGS or REMOVE_ZERO_WIDTH_SPACES

local CHECK_SYMBOL  = "ASCII_SMUGGLING_CHECK"
local RESULT_SYMBOL = "ASCII_SMUGGLING_TAGS"
local HEADER_SYMBOL = "ASCII_SMUGGLING_HEADERS"
local CACHE_KEY = "ascii_smuggling_state_v2"

local HDR_STATUS  = "X-ASCII-Smuggling"
local HDR_TEXT    = "X-ASCII-Smuggling-Text"
local HDR_CONTEXT = "X-ASCII-Smuggling-Context"

-- 診断の表示上限と走査上限
local MAX_FINDINGS = 3
local MAX_SEQ_STORE = 512
local MAX_SEQUENCES = 1000
local MAX_TAG_CHARS_SCAN = 100000
local MAX_TEXT_PER_FINDING = 128
local MAX_CONTEXT_PER_FINDING = 160
local MAX_HEADER_VALUE = 512
local CONTEXT_CODEPOINTS = 12

local TAG_BASE      = 0xE0000
local TAG_MIN       = 0xE0000
local TAG_MAX       = 0xE007F
local LANGUAGE_TAG  = 0xE0001
local TAG_ASCII_MIN = 0xE0020
local TAG_ASCII_MAX = 0xE007E
local CANCEL_TAG    = 0xE007F
local BLACK_FLAG    = 0x1F3F4

local TAG_UTF8_PREFIX = "\243\160"

local VALID_SUBDIVISION_FLAGS = {
  gbeng = true, -- England
  gbsct = true, -- Scotland
  gbwls = true, -- Wales
}

local HEADER_SOURCES = {
  { "Subject",  "subject" },
  { "From",     "from" },
  { "Sender",   "sender" },
  { "Reply-To", "reply_to" },
  { "To",       "to" },
  { "Cc",       "cc" },
}

local SOURCE_ORDER = {
  "subject", "from", "sender", "reply_to",
  "to", "cc", "filename", "body",
}

local SOURCE_TOKEN = {
  subject  = "SOURCE_SUBJECT",
  from     = "SOURCE_FROM",
  sender   = "SOURCE_SENDER",
  reply_to = "SOURCE_REPLY_TO",
  to       = "SOURCE_TO",
  cc       = "SOURCE_CC",
  filename = "SOURCE_FILENAME",
  body     = "SOURCE_BODY",
}

local function is_continuation_byte(b)
  return b and b >= 0x80 and b <= 0xBF
end

-- 不正UTF-8は負のバイト値として扱う
local function utf8_next(s, i)
  local b1 = s:byte(i)
  if not b1 then return nil, i end

  if b1 < 0x80 then
    return b1, i + 1
  end

  local b2 = s:byte(i + 1)

  if b1 >= 0xC2 and b1 <= 0xDF and is_continuation_byte(b2) then
    return (b1 - 0xC0) * 0x40 + (b2 - 0x80), i + 2
  end

  local b3 = s:byte(i + 2)

  if b1 >= 0xE0 and b1 <= 0xEF and
      is_continuation_byte(b2) and is_continuation_byte(b3) then
    if b1 == 0xE0 and b2 < 0xA0 then return -b1, i + 1 end
    if b1 == 0xED and b2 > 0x9F then return -b1, i + 1 end

    return (b1 - 0xE0) * 0x1000 +
           (b2 - 0x80) * 0x40 +
           (b3 - 0x80), i + 3
  end

  local b4 = s:byte(i + 3)

  if b1 >= 0xF0 and b1 <= 0xF4 and
      is_continuation_byte(b2) and
      is_continuation_byte(b3) and
      is_continuation_byte(b4) then
    if b1 == 0xF0 and b2 < 0x90 then return -b1, i + 1 end
    if b1 == 0xF4 and b2 > 0x8F then return -b1, i + 1 end

    return (b1 - 0xF0) * 0x40000 +
           (b2 - 0x80) * 0x1000 +
           (b3 - 0x80) * 0x40 +
           (b4 - 0x80), i + 4
  end

  return -b1, i + 1
end

local function is_tag(cp)
  return cp and cp >= TAG_MIN and cp <= TAG_MAX
end

local function clip(s, max_len)
  if #s <= max_len then return s end
  if max_len <= 3 then return s:sub(1, max_len) end
  return s:sub(1, max_len - 3) .. "..."
end

local function push_ring(ring, cp)
  ring[#ring + 1] = cp
  if #ring > CONTEXT_CODEPOINTS then
    table.remove(ring, 1)
  end
end

-- 診断用の記号と入力文字を区別するためのエスケープ
local STRUCTURAL_ASCII = {
  [0x5C] = true, -- \
  [0x3C] = true, -- <
  [0x3E] = true, -- >
  [0x7C] = true, -- |
  [0x3D] = true, -- =
  [0x3B] = true, -- ;
}

local function render_attacker_ascii(byte)
  if byte == 0x20 then
    return "<SP>"
  end
  if STRUCTURAL_ASCII[byte] then
    return string.format("\\x%02X", byte)
  end
  return string.char(byte)
end

local TAG_CONTEXT_ASCII_NAME = {
  [0x5C] = "BACKSLASH",     -- \
  [0x3C] = "LESS_THAN",     -- <
  [0x3E] = "GREATER_THAN",  -- >
  [0x7C] = "PIPE",          -- |
  [0x3D] = "EQUALS",        -- =
  [0x3B] = "SEMICOLON",     -- ;
}

local function render_context_tag_cp(cp)
  if cp >= TAG_ASCII_MIN and cp <= TAG_ASCII_MAX then
    local byte = cp - TAG_BASE

    if byte == 0x20 then
      return "<TAG_SPACE>"
    end

    local name = TAG_CONTEXT_ASCII_NAME[byte]
    if name then
      return "<TAG:" .. name .. ">"
    end

    return "<TAG:" .. string.char(byte) .. ">"
  end

  if cp == LANGUAGE_TAG then
    return "<LANGUAGE_TAG>"
  end

  if cp == CANCEL_TAG then
    return "<CANCEL_TAG>"
  end

  return string.format("<TAG:U+%X>", cp)
end

local function render_context_cp(cp)
  if cp < 0 then
    return string.format("\\x%02X", -cp)
  end

  if is_tag(cp) then
    return render_context_tag_cp(cp)
  end

  if cp == 0x09 or cp == 0x0A or cp == 0x0D then
    return " "
  end

  if cp >= 0x20 and cp <= 0x7E then
    if STRUCTURAL_ASCII[cp] then
      return string.format("\\x%02X", cp)
    end
    return string.char(cp)
  end

  if cp < 0x20 or cp == 0x7F then
    return string.format("\\x%02X", cp)
  end

  return string.format("\\u{%X}", cp)
end

local function decode_tag_sequence_for_diag(seq, full_len)
  local out = {}

  for _, cp in ipairs(seq) do
    if cp >= TAG_ASCII_MIN and cp <= TAG_ASCII_MAX then
      out[#out + 1] = render_attacker_ascii(cp - TAG_BASE)
    elseif cp == LANGUAGE_TAG then
      out[#out + 1] = "<LANGUAGE_TAG>"
    elseif cp == CANCEL_TAG then
      out[#out + 1] = "<CANCEL_TAG>"
    else
      out[#out + 1] = string.format("<U+%X>", cp)
    end
  end

  if full_len > #seq then
    out[#out + 1] =
      string.format("<SEQ_TRUNCATED:%d>", full_len - #seq)
  end

  return table.concat(out)
end

local function left_context(s, tag_pos)
  local start = math.max(1, tag_pos - (CONTEXT_CODEPOINTS * 4) - 8)

  while start < tag_pos and is_continuation_byte(s:byte(start)) do
    start = start + 1
  end

  local ring = {}
  local i = start

  while i < tag_pos do
    local cp, next_i = utf8_next(s, i)
    if not cp or next_i <= i or next_i > tag_pos then break end
    push_ring(ring, cp)
    i = next_i
  end

  local rendered = {}
  for _, cp in ipairs(ring) do
    rendered[#rendered + 1] = render_context_cp(cp)
  end

  return ring[#ring], table.concat(rendered)
end

local function right_context(s, i)
  local out = {}
  local count = 0

  while i <= #s and count < CONTEXT_CODEPOINTS do
    local cp, next_i = utf8_next(s, i)
    if not cp or next_i <= i then break end

    out[#out + 1] = render_context_cp(cp)

    i = next_i
    count = count + 1
  end

  return table.concat(out)
end

local function is_valid_subdivision_flag(prev_cp, seq, full_len)
  if prev_cp ~= BLACK_FLAG then return false end

  if full_len ~= #seq then return false end

  if #seq < 2 or seq[#seq] ~= CANCEL_TAG then
    return false
  end

  local chars = {}

  for i = 1, #seq - 1 do
    local cp = seq[i]
    if cp < TAG_ASCII_MIN or cp > TAG_ASCII_MAX then
      return false
    end
    chars[#chars + 1] = string.char(cp - TAG_BASE)
  end

  return VALID_SUBDIVISION_FLAGS[table.concat(chars)] == true
end

-- 任意の除去処理（本文・ヘッダー・ファイル名で共用）
local ZWSP_ENTITIES = {
  ZeroWidthSpace = true,
  NegativeVeryThinSpace = true,
  NegativeThinSpace = true,
  NegativeMediumSpace = true,
  NegativeThickSpace = true,
}
local ZWSP_UTF8 = "\226\128\139"
local function may_have_targets(s, html)
  if REMOVE_UNICODE_TAGS and s:find(TAG_UTF8_PREFIX, 1, true) then return true end
  if REMOVE_ZERO_WIDTH_SPACES and s:find(ZWSP_UTF8, 1, true) then return true end
  if not CLEANUP_ENABLED or not html then return false end
  return s:find("&#", 1, true) ~= nil or
    (REMOVE_ZERO_WIDTH_SPACES and
      (s:find("&Zero", 1, true) ~= nil or s:find("&Negative", 1, true) ~= nil))
end
local function body_cp(s, i, html)
  if html and s:sub(i, i) == "&" then
    local _, last, name = s:find("^&([A-Za-z]+);", i)
    if name and ZWSP_ENTITIES[name] then return 0x200B, last + 1 end
  end
  if html and s:sub(i, i + 1) == "&#" then
    local a, b, digits = s:find("^&#[xX]([0-9a-fA-F]+);?", i)
    local base = 16
    if a ~= i then
      a, b, digits = s:find("^&#([0-9]+);?", i)
      base = 10
    end
    if a == i then
      return tonumber(digits, base) or -1, b + 1
    end
  end
  return utf8_next(s, i)
end

local function strip_body_tags(s, html)
  if not may_have_targets(s, html) then return s, 0 end
  local out, start, i, prev, removed = {}, 1, 1, nil, 0
  while i <= #s do
    local cp, next_i = body_cp(s, i, html)
    if cp == 0x200B and REMOVE_ZERO_WIDTH_SPACES then
      out[#out + 1] = s:sub(start, i - 1)
      start, i = next_i, next_i
      removed = removed + 1
      prev = cp
    elseif is_tag(cp) and REMOVE_UNICODE_TAGS then
      local first, seq, count = i, {}, 0
      while is_tag(cp) do
        count = count + 1
        if #seq < 8 then seq[#seq + 1] = cp end
        i = next_i
        cp, next_i = body_cp(s, i, html)
      end
      if not is_valid_subdivision_flag(prev, seq, count) then
        out[#out + 1] = s:sub(start, first - 1)
        start = i
        removed = removed + count
      end
      prev = nil
    else
      prev, i = cp, next_i
    end
  end
  if removed == 0 then return s, 0 end
  out[#out + 1] = s:sub(start)
  return table.concat(out), removed
end

-- 引用符内のセミコロンを分割しない
local function mime_parameters(value)
  local fields, first, quoted, escaped = {}, 1, false, false
  for i = 1, #value do
    local c = value:sub(i, i)
    if escaped then escaped = false
    elseif quoted and c == "\\" then escaped = true
    elseif c == '"' then quoted = not quoted
    elseif not quoted and c == ";" then
      fields[#fields + 1] = value:sub(first, i - 1)
      first = i + 1
    end
  end
  if quoted then error("unclosed MIME parameter quote") end
  fields[#fields + 1] = value:sub(first)
  return fields
end

local function utf8_content_type(value)
  local fields = mime_parameters(value)
  local out = {fields[1]}
  for i = 2, #fields do
    local field = fields[i]
    if not field:match("^%s*$") then
      local attr = field:match("^%s*([^%s=]+)%s*=")
      attr = attr and attr:lower()
      if not attr or (attr ~= "charset" and not attr:match("^charset%*")) then
        out[#out + 1] = field
      end
    end
  end
  out[#out + 1] = ' charset="utf-8"'
  return table.concat(out, ";")
end

local function sanitized_headers(raw, subtype, nl)
  local out, drop, ct, in_ct = {}, false, nil, false
  for line in (raw:gsub("\r\n", "\n") .. "\n"):gmatch("([^\n]*)\n") do
    if line ~= "" then
      if not line:match("^[ \t]") then
        local name = line:match("^([^:]+):")
        name = name and name:lower()
        in_ct = name == "content-type"
        if in_ct then
          ct = line:match("^[^:]+:%s*(.*)$")
        end
        drop = name == "content-type" or name == "content-transfer-encoding"
          or name == "content-length" or name == "content-md5"
      elseif in_ct then
        ct = ct .. " " .. line:gsub("^[ \t]+", "")
      end
      if not drop then out[#out + 1] = line end
    end
  end
  local content_type = utf8_content_type(ct or ('text/' .. subtype))
  out[#out + 1] = 'Content-Type: ' ..
    rspamd_util.fold_header('Content-Type', content_type, nl == "\r\n" and "crlf" or "lf")
  out[#out + 1] = 'Content-Transfer-Encoding: base64'
  return table.concat(out, nl) .. nl .. nl, content_type
end

local function encode_word(s)
  local out, i = {}, 1
  while i <= #s do
    local last = math.min(i + 41, #s)
    while last > i and last < #s and is_continuation_byte(s:byte(last + 1)) do
      last = last - 1
    end
    out[#out + 1] = "=?UTF-8?B?" ..
      tostring(rspamd_util.encode_base64(s:sub(i, last), 0)):gsub("%s", "") .. "?="
    i = last + 1
  end
  return table.concat(out, " ")
end

-- RFC 2047の符号化部分を個別に処理し、アドレス構文を保持
local function clean_header_value(value)
  local removed = 0
  local result = value:gsub("=%?([^?%s]+)%?([bBqQ])%?([^?]*)%?=",
    function(charset, encoding, payload)
      local decoded
      if encoding:lower() == "b" then
        decoded = rspamd_util.decode_base64(payload)
      else
        decoded = rspamd_util.decode_qp((payload:gsub("_", " ")))
      end
      if not decoded then error("MIME word decode failed") end
      if charset:lower() ~= "utf-8" and charset:lower() ~= "utf8" then
        decoded = rspamd_util.to_utf8(decoded, charset)
      end
      if not decoded then error("MIME word charset conversion failed") end
      local clean, count = strip_body_tags(tostring(decoded), false)
      if count == 0 then return nil end -- retain original encoded word exactly
      removed = removed + count
      return encode_word(clean)
    end)
  local count
  result, count = strip_body_tags(result, false)
  return result, removed + count
end

local function sanitize_message_headers(task, state)
  if not CLEANUP_ENABLED then return end
  local changes = {remove = {}, add = {}, order = {}}
  local total, failures = 0, 0
  for _, item in ipairs(HEADER_SOURCES) do
    local name = item[1]
    local headers = task:get_header_full(name) or {}
    local replacements, count, failed = {}, 0, false
    for _, hdr in ipairs(headers) do
      if hdr.value == nil then failed = true; break end
      local value = tostring(hdr.value)
      local candidate = hdr.decoded ~= nil and
        may_have_targets(tostring(hdr.decoded), false) or
        (hdr.decoded == nil and (may_have_targets(value, false) or
          value:find("=%?[^?%s]+%?[bBqQ]%?") ~= nil))
      local ok, clean, n = true, value, 0
      if candidate then ok, clean, n = pcall(clean_header_value, value) end
      if not ok then failed = true; break end
      replacements[#replacements + 1] = {value = clean, order = -1}
      count = count + n
    end
    if failed then
      failures = failures + 1
    elseif count > 0 then
      changes.remove[name] = 0
      changes.add[name] = replacements
      changes.order[#changes.order + 1] = name
      total = total + count
    end
  end
  if total > 0 then lua_mime.modify_headers(task, changes) end
  state.header_cleanup = failures > 0 and
    (total > 0 and "partial-error" or "error") or
    (total > 0 and "removed" or "unchanged")
  state.header_removed_chars = total
end

local function filename_parameter(value, key, filename)
  local fields = mime_parameters(value)
  local out = {fields[1]}
  for i = 2, #fields do
    local field = fields[i]
    if not field:match("^%s*$") then
      local attr = field:match("^%s*([^%s=]+)%s*=")
      attr = attr and attr:lower()
      if not attr or (attr ~= key and not attr:match("^" .. key .. "%*")) then
        out[#out + 1] = field
      end
    end
  end
  local encoded = filename:gsub(".", function(c)
    return string.format("%%%02X", c:byte())
  end)
  if encoded == "" then encoded = "%61%74%74%61%63%68%6D%65%6E%74" end
  local index = 0
  for i = 1, #encoded, 48 do
    out[#out + 1] = " " .. key .. "*" .. index .. "*=" ..
      (index == 0 and "utf-8''" or "") .. encoded:sub(i, i + 47)
    index = index + 1
  end
  return table.concat(out, ";")
end

local function filename_headers(raw, filename)
  local nl = raw:find("\r\n", 1, true) and "\r\n" or "\n"
  local blocks = {}
  for line in (raw:gsub("\r\n", "\n") .. "\n"):gmatch("([^\n]*)\n") do
    if line ~= "" then
      if line:match("^[ \t]") and #blocks > 0 then
        blocks[#blocks] = blocks[#blocks] .. "\n" .. line
      else blocks[#blocks + 1] = line end
    end
  end
  local changed, has_cd = {}, false
  for i, block in ipairs(blocks) do
    local name, value = block:match("^([^:]+):%s*(.*)$")
    if name and (name:lower() == "content-type" or
        name:lower() == "content-disposition") then
      value = value:gsub("\n[ \t]+", " ")
      local key = name:lower() == "content-type" and "name" or "filename"
      name = key == "name" and "Content-Type" or "Content-Disposition"
      value = filename_parameter(value, key, filename)
      changed[name] = value
      blocks[i] = name .. ": " .. rspamd_util.fold_header(name, value, "lf")
      if key == "filename" then has_cd = true end
    end
  end
  if not has_cd then
    changed['Content-Disposition'] = filename_parameter("inline", "filename", filename)
    blocks[#blocks + 1] = 'Content-Disposition: ' ..
      rspamd_util.fold_header('Content-Disposition', changed['Content-Disposition'], "lf")
  end
  local tail = raw:match("([\r\n]*)$") or ""
  return table.concat(blocks, "\n"):gsub("\n", nl) .. tail, changed
end

-- 元メッセージの位置を基準にMIME変更をまとめ、一度だけ適用
local function new_mime_edits()
  return {edits = {}, filenames = {}, top_changes = {}}
end

local function original_message(task, transaction)
  if not transaction.message then
    transaction.message = tostring(task:get_content())
  end
  return transaction.message
end

local function commit_mime_edits(task, transaction)
  local edits = {}
  for _, edit in ipairs(transaction.edits) do
    if not edit.replaced then edits[#edits + 1] = edit end
  end
  if #edits == 0 then return end
  table.sort(edits, function(a, b) return a.first < b.first end)
  local message, out, pos = transaction.message, {}, 1
  for _, edit in ipairs(edits) do
    if edit.first < pos then error("overlapping MIME edits") end
    out[#out + 1] = message:sub(pos, edit.first - 1)
    out[#out + 1] = edit.value
    pos = edit.last + 1
  end
  out[#out + 1] = message:sub(pos)
  local rewritten = table.concat(out)
  if not task:has_header("MIME-Version") then
    local nl = message:find("\r\n", 1, true) and "\r\n" or "\n"
    rewritten = "MIME-Version: 1.0" .. nl .. rewritten
    transaction.top_changes['MIME-Version'] = '1.0'
  end
  if not task:set_message(rewritten) then error("set_message failed") end
  local changes = {add = {}, remove = {}}
  for name, value in pairs(transaction.top_changes) do
    changes.remove[name] = 0
    if value ~= false then changes.add[name] = {value = value, order = 1} end
  end
  if next(changes.remove) then lua_mime.modify_headers(task, changes) end
end

local function sanitize_filenames(task, state, transaction)
  if not CLEANUP_ENABLED then return end
  local standalone = transaction == nil
  transaction = transaction or new_mime_edits()
  state.filename_cleanup = "unchanged"
  if task:get_size() > MAX_SANITIZE_BYTES then
    state.filename_cleanup = "skipped-size"; return
  end
  local edits, top_changes, total, skipped = {}, {}, 0, 0
  for _, part in ipairs(task:get_parts() or {}) do
    local filename = part:get_filename()
    if filename then
      local clean, count = strip_body_tags(tostring(filename), false)
      if count > 0 then
        local parent, protected = part:get_parent(), false
        while parent do
          local t, st = parent:get_type()
          if parent:is_message() or (t == "multipart" and
              (st == "signed" or st == "encrypted")) then protected = true end
          parent = parent:get_parent()
        end
        if protected then skipped = skipped + 1
        else
          local raw = tostring(part:get_raw_headers())
          if not part:get_parent() then raw = tostring(task:get_raw_headers()) end
          local replacement, changed = filename_headers(raw, clean)
          local message = original_message(task, transaction)
          local a, b = message:find(raw, 1, true)
          if raw == "" or not a or message:find(raw, a + 1, true) then
            state.filename_cleanup = "skipped-ambiguous"; return
          end
          if not part:get_parent() then top_changes = changed end
          edits[#edits + 1] = {first = a, last = b, value = replacement}
          total = total + count
        end
      end
    end
  end
  for _, edit in ipairs(edits) do
    transaction.edits[#transaction.edits + 1] = edit
    transaction.filenames[edit.first] = edit
  end
  for name, value in pairs(top_changes) do transaction.top_changes[name] = value end
  if standalone then commit_mime_edits(task, transaction) end
  state.filename_cleanup = skipped > 0 and
    (total > 0 and "partial-protected" or "skipped-protected") or
    (total > 0 and "removed" or "unchanged")
  state.filename_removed_chars = total
end

local function protected_body_part(part)
  local p = part
  while p do
    local t, st = p:get_type()
    if p:is_attachment() or p:is_message() or
        (t == "multipart" and (st == "signed" or st == "encrypted")) then
      return true
    end
    p = p:get_parent()
  end
  return false
end

local function sanitize_body(task, state, transaction)
  if not CLEANUP_ENABLED then return end
  local standalone = transaction == nil
  transaction = transaction or new_mime_edits()
  state.body_cleanup = "unchanged"
  if task:get_size() > MAX_SANITIZE_BYTES then
    state.body_cleanup = "skipped-size"
    return
  end
  local edits, removed, skipped, bytes = {}, 0, 0, 0
  local top_ct
  for _, part in ipairs(task:get_parts() or {}) do
    local t, st = part:get_type()
    if part:is_text() and t == "text" and (st == "plain" or st == "html") then
      local content = part:get_text():get_content("raw_utf")
      if content then
        local text = tostring(content)
        bytes = bytes + #text
        if bytes > MAX_SANITIZE_BYTES then
          state.body_cleanup = "skipped-size"
          return -- atomic: no partial edits
        end
        local clean, count = strip_body_tags(text, st == "html")
        if count > 0 and protected_body_part(part) then
          skipped = skipped + 1
        elseif count > 0 then
          local message = original_message(task, transaction)
          local raw = tostring(part:get_raw_content())
          local a, b = message:find(raw, 1, true)
          if raw == "" or not a or message:find(raw, a + 1, true) then
            state.body_cleanup = "skipped-ambiguous"
            return
          end
          local headers, first
          if not part:get_parent() then
            headers, first = message:sub(1, a - 1), 1
          else
            local rh = tostring(part:get_raw_headers())
            for _, gap in ipairs({"", "\r\n", "\n", "\r\n\r\n", "\n\n"}) do
              local candidate = rh .. gap
              local pos = a - #candidate
              if #rh > 0 and pos > 0 and message:sub(pos, a - 1) == candidate then
                headers, first = candidate, pos
                break
              end
            end
          end
          if not headers then
            state.body_cleanup = "skipped-layout"
            return
          end
          local filename_edit = transaction.filenames[first]
          if filename_edit then
            if filename_edit.first ~= first or filename_edit.last >= a then
              state.body_cleanup = "skipped-layout"; return
            end
            headers = filename_edit.value .. message:sub(filename_edit.last + 1, a - 1)
          end
          local nl = headers:find("\r\n", 1, true) and "\r\n" or "\n"
          local encoded = tostring(rspamd_util.encode_base64(clean))
            :gsub("%s", "")
          local lines = {}
          for n = 1, #encoded, 76 do lines[#lines + 1] = encoded:sub(n, n + 75) end
          local replacement, ct = sanitized_headers(headers, st, nl)
          if not part:get_parent() then top_ct = ct end
          replacement = replacement .. table.concat(lines, nl) .. nl
          edits[#edits + 1] = {first = first, last = b, value = replacement,
            filename_edit = filename_edit}
          removed = removed + count
        end
      end
    end
  end
  if #edits == 0 then
    if skipped > 0 then state.body_cleanup = "skipped-protected" end
    return
  end
  for _, edit in ipairs(edits) do
    if edit.filename_edit then edit.filename_edit.replaced = true end
    transaction.edits[#transaction.edits + 1] = edit
  end
  if top_ct then
    transaction.top_changes['Content-Type'] = top_ct
    transaction.top_changes['Content-Transfer-Encoding'] = 'base64'
    transaction.top_changes['Content-Length'] = false
    transaction.top_changes['Content-MD5'] = false
  end
  if standalone then commit_mime_edits(task, transaction) end
  state.body_cleanup = skipped > 0 and "partial-protected" or "removed"
  state.body_removed_chars = removed
end

local function has_tag_html_entity(s)
  if not s or not s:find("&#", 1, true) then
    return false
  end

  for hex in s:gmatch("&#[xX]([0-9A-Fa-f]+);?") do
    local cp = tonumber(hex, 16)
    if cp and cp >= TAG_MIN and cp <= TAG_MAX then
      return true
    end
  end

  for dec in s:gmatch("&#([0-9]+);?") do
    local cp = tonumber(dec, 10)
    if cp and cp >= TAG_MIN and cp <= TAG_MAX then
      return true
    end
  end

  return false
end

local function add_finding(state, source_group, source_label, seq, seq_len, context)
  state.total_sequences = state.total_sequences + 1
  state.total_tag_chars = state.total_tag_chars + seq_len
  state.sources[source_group] = true

  if seq_len > MAX_SEQ_STORE then
    state.long_sequence = true
  end

  if #state.findings < MAX_FINDINGS then
    state.findings[#state.findings + 1] = {
      source = source_label,
      decoded = clip(
        decode_tag_sequence_for_diag(seq, seq_len),
        MAX_TEXT_PER_FINDING
      ),
      context = clip(context, MAX_CONTEXT_PER_FINDING),
    }
  end
end

local function scan_text(s, source_group, source_label, state, meta)
  if not s or s == "" or state.scan_truncated then return end

  if not s:find(TAG_UTF8_PREFIX, 1, true) then return end

  local pos = 1

  while not state.scan_truncated do
    local found = s:find(TAG_UTF8_PREFIX, pos, true)
    if not found then break end

    local cp = utf8_next(s, found)

    if not is_tag(cp) then
      pos = found + 2
    else
      state.scanned_sequences = state.scanned_sequences + 1

      if state.scanned_sequences > MAX_SEQUENCES then
        state.scan_truncated = true
        break
      end

      local seq = {}
      local seq_len = 0
      local j = found

      while j <= #s do
        local tag_cp, tag_next = utf8_next(s, j)
        if not is_tag(tag_cp) then break end

        state.scanned_tag_chars = state.scanned_tag_chars + 1

        if state.scanned_tag_chars > MAX_TAG_CHARS_SCAN then
          state.scan_truncated = true
          break
        end

        seq_len = seq_len + 1
        if #seq < MAX_SEQ_STORE then
          seq[#seq + 1] = tag_cp
        end

        j = tag_next
      end

      if seq_len > 0 then
        local prev_cp, left = left_context(s, found)

        if not is_valid_subdivision_flag(prev_cp, seq, seq_len) then
          if meta and meta.html_entity then
            state.html_entity = true
          end

          local decoded = decode_tag_sequence_for_diag(seq, seq_len)
          local marker

          if seq_len == 1 and seq[1] == 0xE0020 then
            marker = "<TAG_SPACE>"
          else
            marker = "<TAGS:" .. clip(decoded, 64) .. ">"
          end

          local context = ""
          if #state.findings < MAX_FINDINGS then
            context = left .. marker .. right_context(s, j)
          end

          add_finding(
            state, source_group, source_label,
            seq, seq_len, context
          )
        end
      end

      if state.scan_truncated then break end
      pos = j
    end
  end
end

local function scan_header(task, header_name, source_group, state)
  if state.scan_truncated then return end

  local headers = task:get_header_full(header_name)
  if not headers then return end

  for n, hdr in ipairs(headers) do
    if state.scan_truncated then break end

    local value = hdr.decoded or hdr.value
    if value then
      local label = source_group
      if #headers > 1 then
        label = string.format("%s#%d", source_group, n)
      end

      scan_text(
        tostring(value),
        source_group,
        label,
        state
      )
    end
  end
end

-- HTMLは実文字と文字参照の二重カウントを避ける
local function scan_body(task, state)
  local parts = task:get_text_parts()
  if not parts then return end

  for n, part in ipairs(parts) do
    if state.scan_truncated then break end

    local raw = part:get_content("raw_utf")
    if raw then
      local s = tostring(raw)

      local meta = nil

      if part:is_html() then
        local entity_tag_present =
          has_tag_html_entity(s)

        local ok, decoded =
          pcall(rspamd_util.decode_html_entities, s)

        if ok and decoded then
          s = tostring(decoded)
        end

        if entity_tag_present then
          meta = { html_entity = true }
        end
      end

      scan_text(
        s,
        "body",
        string.format("body#%d", n),
        state,
        meta
      )
    end
  end
end

local function scan_filenames(task, state)
  local parts = task:get_parts()
  if not parts then return end

  for n, part in ipairs(parts) do
    if state.scan_truncated then break end

    local filename = part:get_filename()
    if filename and filename ~= "" then
      scan_text(
        tostring(filename),
        "filename",
        string.format("filename#%d", n),
        state
      )
    end
  end
end

local function build_policy_options(state)
  local options = {}

  if state.total_sequences > 0 then
    options[#options + 1] = "DETECTED"
  end

  for _, source in ipairs(SOURCE_ORDER) do
    if state.sources[source] then
      options[#options + 1] = SOURCE_TOKEN[source]
    end
  end

  if state.html_entity then
    options[#options + 1] = "HTML_ENTITY"
  end

  if state.long_sequence then
    options[#options + 1] = "LONG_SEQUENCE"
  end

  if state.scan_truncated then
    options[#options + 1] = "SCAN_TRUNCATED"
  end

  return options
end

local function sources_to_string(state)
  local result = {}

  for _, source in ipairs(SOURCE_ORDER) do
    if state.sources[source] then
      result[#result + 1] = source
    end
  end

  return table.concat(result, ",")
end

local function build_text_header(state)
  local out = {}

  for _, finding in ipairs(state.findings) do
    out[#out + 1] = finding.source .. "=" .. finding.decoded
  end

  return clip(table.concat(out, " | "), MAX_HEADER_VALUE)
end

local function build_context_header(state)
  local out = {}

  for _, finding in ipairs(state.findings) do
    out[#out + 1] = finding.source .. "=" .. finding.context
  end

  return clip(table.concat(out, " | "), MAX_HEADER_VALUE)
end

local function build_status_header(state)
  local status

  if state.total_sequences > 0 then
    status = string.format(
      "yes; source=%s; sequences=%d; chars=%d",
      sources_to_string(state),
      state.total_sequences,
      state.total_tag_chars
    )
  elseif state.scan_truncated then
    status = string.format(
      "scan-truncated; scanned_sequences=%d; scanned_chars=%d",
      state.scanned_sequences,
      state.scanned_tag_chars
    )
  else
    status = "no"
  end

  if state.html_entity then
    status = status .. "; html_entity=yes"
  end

  if state.long_sequence then
    status = status .. "; long_sequence=yes"
  end

  if state.scan_truncated then
    status = status .. "; truncated=yes"
  end

  if state.total_sequences > #state.findings then
    status = status .. string.format("; shown=%d", #state.findings)
  end

  if state.body_cleanup then
    status = status .. "; body_cleanup=" .. state.body_cleanup
    if state.body_removed_chars then
      status = status .. "; removed_chars=" .. state.body_removed_chars
    end
  end
  for _, area in ipairs({"header", "filename"}) do
    if state[area .. "_cleanup"] then
      status = status .. "; " .. area .. "_cleanup=" .. state[area .. "_cleanup"]
      if (state[area .. "_removed_chars"] or 0) > 0 then
        status = status .. "; " .. area .. "_removed_chars=" .. state[area .. "_removed_chars"]
      end
    end
  end
  return clip(status, MAX_HEADER_VALUE)
end

-- 検出段階では結果を保存し、固定tokenのみをポリシー判定に渡す
local function ascii_smuggling_check(task)
  local state = {
    total_sequences = 0,
    total_tag_chars = 0,
    scanned_sequences = 0,
    scanned_tag_chars = 0,
    findings = {},
    sources = {},
    html_entity = false,
    long_sequence = false,
    scan_truncated = false,
  }

  for _, item in ipairs(HEADER_SOURCES) do
    scan_header(task, item[1], item[2], state)
  end

  scan_filenames(task, state)
  scan_body(task, state)

  task:cache_set(CACHE_KEY, state)

  local options = build_policy_options(state)

  if #options > 0 then
    task:insert_result(
      RESULT_SYMBOL,
      1.0,
      unpack_fn(options)
    )
  end

  return false
end

local function cleanup_skip_reason(task)
  if CLEANUP_INBOUND_ONLY then
    local user = task:get_user()
    if user and user ~= "" then return "skipped-authenticated" end
    local ip = task:get_from_ip()
    if not ip or not ip:is_valid() then return "skipped-unknown-source" end
    if ip:is_local() then return "skipped-local" end
  end
  -- 自サーバーの署名を保護（署名シンボル名を変更した場合はここも合わせる）
  if not IGNORE_SIGNATURE_PROTECTION and
      (task:has_symbol("DKIM_SIGNED") or task:has_symbol("ARC_SIGNED")) then
    return "skipped-signed"
  end
end

-- idempotent段階で除去と診断ヘッダー更新を行う
local function ascii_smuggling_headers(task)
  local state = task:cache_get(CACHE_KEY)
  if state and CLEANUP_ENABLED then
    local checked, reason = pcall(cleanup_skip_reason, task)
    if not checked then reason = "skipped-unknown-source" end
    if reason then
      for _, area in ipairs({"body", "filename", "header"}) do
        state[area .. "_cleanup"] = reason
        state[area .. "_removed_chars"] = 0
      end
    else
      local ok, err = pcall(function()
        local transaction = new_mime_edits()
        sanitize_filenames(task, state, transaction)
        sanitize_body(task, state, transaction)
        commit_mime_edits(task, transaction)
      end)
      if not ok then
        state.filename_cleanup, state.body_cleanup = "error", "error"
        state.filename_removed_chars, state.body_removed_chars = 0, 0
        require("rspamd_logger").errx(task, "ASCII smuggling MIME cleanup: %s", err)
      end
      ok, err = pcall(sanitize_message_headers, task, state)
      if not ok then
        state.header_cleanup = "error"
        require("rspamd_logger").errx(task, "ASCII smuggling header cleanup: %s", err)
      end
    end
    task:cache_set(CACHE_KEY, state)
  end

  local has_existing =
    task:has_header(HDR_STATUS) or
    task:has_header(HDR_TEXT) or
    task:has_header(HDR_CONTEXT)

  local has_signal =
    state and (
      state.total_sequences > 0 or
      state.scan_truncated or
      (state.body_removed_chars or 0) > 0 or
      (state.header_removed_chars or 0) > 0 or
      (state.filename_removed_chars or 0) > 0
    )

  if not has_existing and not has_signal then
    return false
  end

  local changes = {
    remove = {
      [HDR_STATUS]  = 0,
      [HDR_TEXT]    = 0,
      [HDR_CONTEXT] = 0,
    },
    order = {
      HDR_STATUS,
      HDR_TEXT,
      HDR_CONTEXT,
    },
  }

  if has_signal then
    local add = {}

    add[HDR_STATUS] = {
      value = build_status_header(state),
      order = 1,
    }

    if #state.findings > 0 then
      add[HDR_TEXT] = {
        value = build_text_header(state),
        order = 1,
      }

      add[HDR_CONTEXT] = {
        value = build_context_header(state),
        order = 1,
      }
    end

    changes.add = add
  end

  lua_mime.modify_headers(task, changes)
  return false
end

-- シンボル登録
local check_id = rspamd_config:register_symbol({
  name = CHECK_SYMBOL,
  type = "callback,mime",
  callback = ascii_smuggling_check,
  score = 0.0,
  group = "phishing",
  description =
    "Detect Unicode Tags potentially used for ASCII smuggling",
})

rspamd_config:register_symbol({
  name = RESULT_SYMBOL,
  type = "virtual",
  parent = check_id,
  score = 0.0,
  group = "phishing",
  description =
    "Unicode Tag / ASCII smuggling scan result",
})

rspamd_config:register_symbol({
  name = HEADER_SYMBOL,
  type = "idempotent",
  flags = "explicit_disable,nostat",
  callback = ascii_smuggling_headers,
  score = 0.0,
  group = "phishing",
  description =
    "Add ASCII smuggling diagnostic headers after filtering",
})

ASCII_SMUGGLING_POLICY {
    description = "ASCII smuggling 検出";
    target_symbol = "ASCII_SMUGGLING_TAGS";
    type = "symbol_options";
    dynamic_symbols = true;
    map = "${LOCAL_CONFDIR}/local.d/ascii_smuggling.map";
    multi = true;
    score = 1.0;
}

DETECTED            ASCII_SMUGGLING:5.0

SOURCE_FROM         ASCII_SMUGGLING_FROM:3.0
SOURCE_SENDER       ASCII_SMUGGLING_SENDER:3.0
SOURCE_REPLY_TO     ASCII_SMUGGLING_REPLY_TO:3.0

SOURCE_SUBJECT      ASCII_SMUGGLING_SUBJECT:2.0
SOURCE_TO           ASCII_SMUGGLING_TO:1.0
SOURCE_CC           ASCII_SMUGGLING_CC:1.0

SOURCE_FILENAME     ASCII_SMUGGLING_FILENAME:2.0
SOURCE_BODY         ASCII_SMUGGLING_BODY:0.0

HTML_ENTITY         ASCII_SMUGGLING_HTML_ENTITY:2.0
LONG_SEQUENCE       ASCII_SMUGGLING_LONG_SEQUENCE:3.0
SCAN_TRUNCATED      ASCII_SMUGGLING_SCAN_TRUNCATED:5.0

#!/usr/bin/env python3
"""Send one harmless Unicode Tags / zero-width space test message via SMTP."""

import argparse
import getpass
import smtplib
import ssl
import sys
from email.message import EmailMessage
from email.policy import SMTP
from email.utils import formatdate, make_msgid
from pathlib import Path

TAG_BASE = 0xE0000
TAG_MIN = 0xE0000
TAG_MAX = 0xE007F
ZWSP = "\u200b"


def tags(text: str) -> str:
    """Encode printable ASCII as Unicode Tags."""
    if any(not 0x20 <= ord(char) <= 0x7E for char in text):
        raise ValueError("tags() accepts printable ASCII only")
    return "".join(chr(TAG_BASE + ord(char)) for char in text)


def build_message(sender: str, recipient: str) -> EmailMessage:
    # The hidden text is diagnostic prose only. It contains no executable command.
    body = (
        "ASCII smuggling detection test\n\n"
        "Visible split word: fun" + tags(" ") + "ding\n"
        "Hidden tag text follows: " + tags("harmless test") + "\n"
        "Four findings: "
        + " ".join("x" + tags(char) for char in "abcd")
        + "\n"
        "ZWSP split word: zero" + ZWSP + "width\n"
        "ZWSP boundaries: " + ZWSP + "middle" + ZWSP + "\n"
    )

    message = EmailMessage(policy=SMTP)
    message["From"] = sender
    message["To"] = recipient
    message["Subject"] = "ASCII smuggling detection test"
    message["Date"] = formatdate(localtime=True)
    message["Message-ID"] = make_msgid(domain="ascii-smuggling-test.invalid")
    message["Auto-Submitted"] = "auto-generated"
    message["X-ASCII-Smuggling-Test"] = "harmless-local-test"
    # Keep invisible characters inside an ASCII-only Base64 MIME body.
    message.set_content(body, charset="utf-8", cte="base64")
    return message


def validate_message(message: EmailMessage) -> tuple[int, int]:
    decoded = message.get_content()
    count = sum(TAG_MIN <= ord(char) <= TAG_MAX for char in decoded)
    if count != 18:
        raise RuntimeError(f"internal validation failed: expected 18 Tags, got {count}")
    zwsp_count = decoded.count(ZWSP)
    if zwsp_count != 3:
        raise RuntimeError(f"internal validation failed: expected 3 ZWSP, got {zwsp_count}")
    return count, zwsp_count


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Send an ASCII smuggling test message to your own mail server."
    )
    parser.add_argument("recipient", help="recipient address on the server under test")
    parser.add_argument("--host", default="127.0.0.1", help="SMTP host (default: 127.0.0.1)")
    parser.add_argument("--port", type=int, default=25, help="SMTP port (default: 25)")
    parser.add_argument(
        "--from-address",
        help="sender address (default: recipient address)",
    )
    parser.add_argument("--starttls", action="store_true", help="use SMTP STARTTLS")
    parser.add_argument("--username", help="SMTP username; password is prompted securely")
    parser.add_argument(
        "--output",
        type=Path,
        help="write an .eml file instead of sending it",
    )
    args = parser.parse_args()
    args.recipient = args.recipient.strip()
    if not args.recipient:
        parser.error("recipient address must not be empty")
    if args.from_address is None:
        args.from_address = args.recipient
    else:
        args.from_address = args.from_address.strip()
        if not args.from_address:
            parser.error("sender address must not be empty")
    return args


def main() -> int:
    args = parse_args()
    message = build_message(args.from_address, args.recipient)
    tag_count, zwsp_count = validate_message(message)

    if args.output:
        args.output.write_bytes(message.as_bytes())
        print(f"Wrote {args.output} ({tag_count} Unicode Tags, {zwsp_count} ZWSP)")
        return 0

    try:
        with smtplib.SMTP(args.host, args.port, timeout=15) as smtp:
            smtp.ehlo()
            if args.starttls:
                smtp.starttls(context=ssl.create_default_context())
                smtp.ehlo()
            if args.username:
                smtp.login(args.username, getpass.getpass("SMTP password: "))
            refused = smtp.send_message(
                message,
                from_addr=args.from_address,
                to_addrs=[args.recipient],
            )
    except (OSError, smtplib.SMTPException) as exc:
        print(f"SMTP send failed: {exc}", file=sys.stderr)
        return 1

    if refused:
        print(f"Recipient refused: {refused}", file=sys.stderr)
        return 1

    print(f"Sent: {message['Message-ID']}")
    print(f"Embedded Unicode Tags: {tag_count}")
    print(f"Embedded zero-width spaces (U+200B): {zwsp_count}")
    print("Check X-ASCII-Smuggling* headers in the delivered message.")
    print("Expected body removed_chars: Tags only=18, ZWSP only=3, both=21.")
    print("SMTP acceptance does not confirm delivery or Rspamd filtering.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

X-ASCII-Smuggling-Context: body#1=it word: fun<TAG_SPACE>ding  Hidden | body#1=xt follows: <TAGS:harmless<SP>test>  Four findi | body#1= findings: x<TAGS:a> x<TAG:b> x<TAG:c> x<TAG:d>  Z
X-ASCII-Smuggling-Text: body#1=<SP> | body#1=harmless<SP>test | body#1=a
X-ASCII-Smuggling: yes; source=body; sequences=6; chars=18; shown=3
X-ASCII-Smuggling-Test: harmless-local-test

X-ASCII-Smuggling-Context: body#1=it word: fun<TAG_SPACE>ding  Hidden | body#1=xt follows: <TAGS:harmless<SP>test>  Four findi | body#1= findings: x<TAGS:a> x<TAG:b> x<TAG:c> x<TAG:d>  Z
X-ASCII-Smuggling-Text: body#1=<SP> | body#1=harmless<SP>test | body#1=a
X-ASCII-Smuggling: yes; source=body; sequences=6; chars=18; shown=3; body_cleanup=removed; removed_chars=21; header_cleanup=unchanged; filename_cleanup=unchanged
X-ASCII-Smuggling-Test: harmless-local-test