throttle spam filter relay messages

5 files changed, 146 insertions, 50 deletions

diff --git a/labs_change_detector b/labs_change_detector
index 804b403..dc2b405 100755
--- a/labs_change_detector
+++ b/labs_change_detector
@@ -9,19 +9,9 @@ then
     echo '[LABS]: inotify is a dep. fail'
     exit
 fi
-if [[ -d "${IPC_NOTICE_CHANNEL}" ]]
+if [[ ! -d "${BOT_MAIL_DIR}" ]]
 then
-    echo '[LABS]: no such file set in IPC_NOTICE_CHANNEL: '${IPC_NOTICE_CHANNEL}'. fail'
-    exit
-fi
-if [[ -d "${BOT_MAIL_DIR}" ]]
-then
-    echo '[LABS]: no such directory set in BOT_MAIL_DIR: '${BOT_MAIL_DIR}'. fail'
-    exit
-fi
-if [[ -d "${IPC_STORE_FILE}" ]]
-then
-    echo '[LABS]: no such file set in IPC_STORE_FILE: '${IPC_STORE_FILE}'. fail'
+    echo "[LABS]: no such directory set in BOT_MAIL_DIR: '${BOT_MAIL_DIR}'. fail"
     exit
 fi
 
diff --git a/lib/main.sh b/lib/main.sh
index b51716a..ac28f05 100644
--- a/lib/main.sh
+++ b/lib/main.sh
@@ -334,7 +334,7 @@ debug_init
 
 log_info_stdout "Loading transport"
 source "${config_transport_dir}/${config_transport}.sh"
-source "${config_transport_dir}/transport_plus_ipc.sh.inc"
+source "${config_transport_dir}/transport_plus_ipc.sh.inc" # common transport_read_line()
 if [[ $? -ne 0 ]]; then
 	log_fatal "Couldn't load transport. Couldn't load the file..."
 	envbot_quit 2
diff --git a/lib/send.sh b/lib/send.sh
index accdc5f..59f90df 100644
--- a/lib/send.sh
+++ b/lib/send.sh
@@ -179,10 +179,16 @@ send_quit() {
 
 send_ipc_msg()
 {
-  local delay_secs=$1 ; shift ;
-  local delivery_time=$(( $(date +%s) + ${delay_secs} ))
-  local message=$*
-
-  echo "${delivery_time},${IPC_NOTICE_CHANNEL},${message}" >> "${IPC_STORE_FILE}"
-  sort -o "${IPC_STORE_FILE}" "${IPC_STORE_FILE}"
+  if [[ -z "${IPC_NOTICE_CHANNEL}" ]]
+  then echo '[SEND]: no target channel set in IPC_NOTICE_CHANNEL'
+  elif ! touch "${IPC_STORE_FILE}" 2> /dev/null
+  then echo "[SEND]: no such file set in IPC_STORE_FILE: '${IPC_STORE_FILE}'"
+  else
+    local delay_secs=$1 ; shift ;
+    local message=$*
+    local delivery_time=$(( $(date +%s) + ${delay_secs} ))
+
+    echo "${delivery_time} ${IPC_NOTICE_CHANNEL} ${message}" >> "${IPC_STORE_FILE}"
+    sort -o "${IPC_STORE_FILE}" "${IPC_STORE_FILE}"
+  fi
 }
diff --git a/modules/m_spamfilter.sh b/modules/m_spamfilter.sh
index f0f0ca3..0c3b21b 100644
--- a/modules/m_spamfilter.sh
+++ b/modules/m_spamfilter.sh
@@ -33,6 +33,13 @@
 readonly RELAY_NICK='a-user'
 readonly II_ROOT_DIR=/home/pbot/irc
 readonly IS_RELAY_BOT_REGISTERED=0
+readonly NOOB_MSG_DELAY_SECS=0 # 0 to disable
+readonly THROTTLE_SECS=60      # 0 to disable
+
+# user messages
+readonly NOOB_MSG_PREFIX='Welcome, _NICK_'
+readonly NOOB_MSG="! If you are new to this channel, please read the private message I sent you."
+readonly THROTTLED_MSG="Due to the recent spam-storm, chat from unregistered users is being filtered and throttled. Your messages will not be visible to others unless I relay them for you. If you send more than one message within ${THROTTLE_SECS} seconds, I will not relay them all. I will also not relay web links or words that confuse me. If you do not see me repeat your message, please wait ${THROTTLE_SECS} seconds and then send that message again; or register your nick to remove these restrictions."
 
 # generic spam-like regexes
 readonly ASCII_SET_REGEX='[^ -~]'
@@ -84,7 +91,7 @@ readonly BOT_PASS="${config_server_passwd}"
 module_spamfilter_INIT()
 {
   modinit_API='2'
-  modinit_HOOKS='on_JOIN on_PRIVMSG'
+  modinit_HOOKS='on_JOIN on_PART on_PRIVMSG'
   helpentry_module_spamfilter_description="Provides support for filtering known spam using +z mode."
 
   which ii &>/dev/null || log_error "[SPAMFILTER]: ERROR: module failed to load - cannot find the \`ii\` program"
@@ -133,6 +140,20 @@ DBG_SPAMFILTER_RELAY_USER
     fi
 }
 
+module_spamfilter_on_PART()
+{
+  local whoparted=
+  local channel=$2
+  parse_hostmask_nick "$1" 'whoparted'
+  local welcome=".*${NOOB_MSG_PREFIX}${whoparted}$"
+  local person_dir=$(people_dir ${whoparted})
+
+  [[ " ${FILTER_CHANNELS} " =~ " ${channel} " ]]            && \
+  [[ " ${OP_CHANNELS} "     =~ " ${channel} " ]]            && \
+  [[ -d "${person_dir}"                       ]]            && \
+  grep -E ${welcome_regex} "${IPC_STORE_FILE}" &> /dev/null && rm -rf ${person_dir}
+}
+
 module_spamfilter_on_PRIVMSG() # (hostmask , target , query)
 {
   local sender
@@ -143,10 +164,19 @@ module_spamfilter_on_PRIVMSG() # (hostmask , target , query)
 
 DBG_SPAMFILTER_CRITERIA
 
-  # ignore internal messages and chat from registered users
-  if ! is_filtered_channel "${target}"            || \
-       is_internal_user    "${sender}"            || \
-       is_public_chat      "${sender}" "${query}"
+  # post delayed welcome message to new unregistered user
+  if   is_internal_user "${sender}" && [[ "${query}" =~ ^"${NOOB_MSG_PREFIX}"* ]]
+  then local noob=${query##${NOOB_MSG_PREFIX}}
+       local noob_dir=$(people_dir ${nick})
+
+       [[ -d "${noob_dir}" ]] && send_msg "${target}" "${query//_NICK_/}${NOOB_MSG}"
+
+       was_handled=1
+
+  # ignore other internal messages and chat from registered users
+  elif ! is_filtered_channel "${target}"            || \
+         is_internal_user    "${sender}"            || \
+         is_public_chat      "${sender}" "${query}"
   then was_handled=0
 
   # ignore chat that is known spam or otherwise nonsense
@@ -155,17 +185,31 @@ DBG_SPAMFILTER_CRITERIA
 
 DBG_SPAMFILTER
 
-  # relay chat from an unregistered user to the channel
-  else local message="(${sender} said): ${query}"
+  # handle otherwise legitimate chat from unregistered users
+  else # greet unregistered users (remind them to read THROTTLED_MSG)
+       is_noob ${sender} && (( ${NOOB_MSG_DELAY_SECS} > 0 ))             && \
+       send_ipc_msg ${NOOB_MSG_DELAY_SECS} "${NOOB_MSG_PREFIX}${sender}"
 
-       if   (( ${IS_RELAY_BOT_REGISTERED} ))
-       then echo                 "${message}" > ${II_DIR}/${target}/in
-       else send_msg "${target}" "${message}"
-       fi
+       # throttle chat from unregistered users
+       if   is_throttled ${sender}
+       then # notify unregistered users of the chat throttle
+            send_msg "${sender}" "${THROTTLED_MSG}"
+
+            was_handled=1
+
+       # relay chat from an unregistered user to the channel
+       else local message="(${sender} said): ${query}"
+
+            if   (( ${IS_RELAY_BOT_REGISTERED} ))
+            then echo                 "${message}" > ${II_DIR}/${target}/in
+            else send_msg "${target}" "${message}"
+            fi
+
+            was_handled=0
 
 DBG_SPAMFILTER_RELAY_CHAT
+       fi
 
-       was_handled=0
   fi
 
   # supress or allow further handling of this message
@@ -229,6 +273,48 @@ is_spam() # (chat_msg)
   return 1
 }
 
+people_dir() # (sender)
+{
+  local nick=$1
+  declare -l nick_lower=${nick//\/}
+  local nick_dir=announcements/people/${nick_lower}
+
+  echo ${nick_dir}
+}
+
+is_noob() # (sender)
+{
+  local nick=$1
+  local person_dir=$(people_dir ${nick})
+
+  [[ ! -f "${person_dir}/seen" ]]
+}
+
+is_throttled() # (sender)
+{
+  (( ${THROTTLE_SECS} > 0 )) || return false
+
+  local sender=$1
+  local sender_dir=$(people_dir ${sender})
+  local the_time_now=$(date +%s)
+  local should_throttle
+
+  if   is_noob ${sender}
+  then # suppress first ever message from this nick
+       should_throttle=1
+
+       mkdir ${sender_dir}
+  else # suppress any message sent too soon since last
+       local last_seen_time=$(stat -c %Y ${sender_dir}/seen 2> /dev/null)
+       should_throttle=$(( ${last_seen_time} + ${THROTTLE_SECS} > ${the_time_now} ))
+  fi
+
+  # denote this nick as not is_noob()
+  touch ${sender_dir}/seen
+
+  (( ${should_throttle} ))
+}
+
 
 ## DEBUG ##
 
@@ -240,14 +326,28 @@ DBG_SPAMFILTER_CRITERIA()
 {
   (( ${DEBUG} )) || return
 
-  echo -n "[SPAMFILTER]: target='${target}'" ; ! is_filtered_channel "${target}"            && echo    " => wrong channel - ignoring"        && return ;
-  echo -n "[SPAMFILTER]: sender='${sender}'" ;   is_internal_user    "${sender}"            && echo    " => from internal user - ignoring"   && return ;
-  echo -n "[SPAMFILTER]: query='${query}'"   ;   is_public_chat      "${sender}" "${query}" && echo    " => from registered user - ignoring" && return ;
-                                                 is_nonsense         "${query}"             && echo -n " => is nonsense"                               ;
-                                               ! is_shared_lib_error "${query}"             && echo -n " => is not shared lib error"                   ;
-                                                (is_nonsense         "${query}" && \
-                                               ! is_shared_lib_error "${query}"  )          && echo    " - suppressing"                      && return ;
-                                                 is_spam             "${query}"             && echo    " => is known spam - suppressing"     && return ;
+  # store "last seen" state to restore after is_throttled() clobbers
+  is_noob ${sender} && local was_noob=1 || local was_noob=0
+  local last_seen_time="$(stat -c %y $(people_dir ${sender})/seen 2> /dev/null)"
+
+  echo "[SPAMFILTER]: sender='${sender}' target='${target}' query='${query}'" ; local pad='              guard ' ;
+  echo -n "${pad}1: " ;  (is_internal_user    "${sender}"           && \
+                          [[ "${query}" =~ ^"${NOOB_MSG_PREFIX}"* ]] ) && echo    " => is welcome msg - posting (maybe)"      && return || echo 'n/a' ;
+
+  echo -n "${pad}2: " ; ! is_filtered_channel "${target}"              && echo    " => wrong channel - ignoring"              && return               ;
+                          is_internal_user    "${sender}"              && echo    " => from internal user - ignoring"         && return               ;
+                          is_public_chat      "${sender}" "${query}"   && echo    " => from registered user - ignoring"       && return ;  echo 'n/a' ;
+
+  echo -n "${pad}3: " ;   is_nonsense         "${query}"               && echo -n " => is nonsense"                                                   ;
+                         (is_nonsense         "${query}" && \
+                        ! is_shared_lib_error "${query}"  )            && echo    " => is not shared lib error - suppressing" && return               ;
+                          is_spam             "${query}"               && echo    " => is known spam - suppressing"           && return ;  echo 'n/a' ;
+
+  echo -n "${pad}4: " ;   is_noob             "${sender}"              && echo -n " => is noob - greeting"                                            ;
+                          is_throttled        "${sender}"              && echo    " => is throttled - suppressing"                      || echo 'OK'  ;
+
+  # restore "last seen" state
+  (( ${was_noob} )) && rm -rf $(people_dir ${sender}) || touch -md "${last_seen_time}" $(people_dir ${sender})/seen
 }
 DBG_SPAMFILTER()            { (( ${DEBUG} )) && echo "[SPAMFILTER]: !!!triggered!!! sender=${sender}" ; }
 DBG_SPAMFILTER_RELAY_CHAT() { (( ${DEBUG} )) && echo "[SPAMFILTER]: relaying chat from unregistered user sender=${sender}" ; }
diff --git a/transport/transport_plus_ipc.sh.inc b/transport/transport_plus_ipc.sh.inc
index b4eebd4..fc9ab70 100644
--- a/transport/transport_plus_ipc.sh.inc
+++ b/transport/transport_plus_ipc.sh.inc
@@ -1,34 +1,34 @@
-if [[ -d "${IPC_STORE_FILE}" ]]
-then
-    echo '[IPC]: no such file set in IPC_STORE_FILE: '${IPC_STORE_FILE}'. fail'
-    exit
-fi
-
 
 # set $line variable for further processing
 transport_plus_ipc_read_line() {
+    local postpone_ipc=0
     while true
     do
-        local n_msgs="$(wc -l "${IPC_STORE_FILE}" 2> /dev/null | cut -d ' ' -f 1)"
-
+        local n_msgs=$(wc -l "${IPC_STORE_FILE}" 2> /dev/null | cut -d ' ' -f 1)
         # set $line variable from IPC message
-        if [[ -r "${IPC_STORE_FILE}" ]] && [[ -w "${IPC_STORE_FILE}" ]] && (( ${n_msgs} ))
+        if ! (( ${postpone_ipc} )) && (( ${n_msgs} )) && [[ -w "${IPC_STORE_FILE}" ]]
         then
             read -r line < "${IPC_STORE_FILE}"
 
-            if [[ "${line}" =~ ^([0-9]{10}),(#.*),(.*)$ ]]
+            if [[ "${line}" =~ ^([0-9]{10})\ (#[^\ ]*)\ (.*)$ ]]
             then
                 local delivery_time="${BASH_REMATCH[1]}"
                 local target_channel="${BASH_REMATCH[2]}"
                 local message="${BASH_REMATCH[3]}"
                 local pending_line="${IPC_INJECT_PREFIX} ${target_channel} :${message}"
 
+# echo "[IPC]: delivery_time=${BASH_REMATCH[1]}"
+# echo "[IPC]: target_channel=${BASH_REMATCH[2]}"
+# echo "[IPC]: message=${BASH_REMATCH[3]}"
+# echo "[IPC]: pending_line=${IPC_INJECT_PREFIX} ${target_channel} :${message}"
+(( ${delivery_time} < $(date +%s) )) && echo "[IPC]: relaying" # || echo "[IPC]: postponing"
+
                 (( ${delivery_time} < $(date +%s) )) && line="${pending_line}" || line=''
             else
                 line='ERROR: invalid IPC message'
             fi
 
-            [[ -n "${line}" ]] || break
+            [[ -z "${line}" ]] && postpone_ipc=1 && continue
 
             if (( ${n_msgs} < 2 ))
             then