Compare commits

..

No commits in common. "1a366ee3eb8dcf4f072b8dd4359a3a33abaad4e2" and "99c1495d3f193ce122061a89c9cc657b741acdca" have entirely different histories.

10 changed files with 66 additions and 338 deletions

View File

@ -1 +0,0 @@
3.1.2

View File

@ -15,5 +15,5 @@ DB_URL="sqlite://dev.db"
PORT=10081
DEBUG=false
CPU_SLEEP=2
CPU_CYCLE=1
LOG_FILE=/var/log/mailinglistrb.log

View File

@ -1,20 +1,14 @@
class Distributor
class Attributes < Hash
# Look at the subject if there are attributes parts.
# If not, look at the body too.
# TODO: this part can be improved with smarter attr detection
def self.parse(subject:, body:)
new = Attributes.new
subject_parts = subject.to_s.split(",")
if !subject_parts.empty?
subject_parts.each do
new.add_attribute!(_1)
end
elsif body.include?("=")
body.to_s.split("\r\n").each { new.add_attribute!(_1) }
subject_added_attribute = false
subject.to_s.split(",").each do
new.add_attribute!(_1)
subject_added_attribute = true
end
body.to_s.split("\r\n").each { new.add_attribute!(_1) } if body.include?("=") && subject_added_attribute == false
new
end
@ -35,13 +29,9 @@ class Distributor
"help" => Actions::Help.new(distributor: self),
"set-permissions" => Actions::SetPermissions.new(distributor: self),
"list-users" => Actions::ListUsers.new(distributor: self),
"validate" => Actions::ValidateDistribute.new(distributor: self),
"refuse" => Actions::RefuseDistribute.new(distributor: self),
}
end
attr_reader :imap_client, :smtp_client, :handlers if $debug
# Make sure any incoming email is properly directed to the right action.
def handle_one(mail)
$logger.info "incoming email from #{mail.from} | #{mail.subject}"
list = Mailinglist.search_mail(mail)
@ -49,31 +39,23 @@ class Distributor
subject, subject_attributes = mail.subject.split(",", 2)
attributes = Attributes.parse(subject: subject_attributes, body: mail.body)
handler = @handlers[subject] || @handlers[:default]
$logger.info "#{handler.class}#handle on <#{list.email}> for <#{mail.from}>"
handler.handle(list:, mail:, attributes:)
$logger.info "#{handler.class}#handle on #{list.email} for #{mail.from}"
handler.handle(list: list, mail: mail, attributes: attributes)
else
$logger.warn "list #{mail.to} do not exist (asked by #{mail.from})"
end
mail.seen!(imap_client: @imap_client)
end
# alias for {Protocols::Smtp#distribute}
def distribute(*ary, **opt)
@smtp_client.distribute(*ary, **opt)
end
# Run the main loop that read all incoming emails.
def start(cpu_sleep: 1)
$logger.info "fetching new mail to distribute every #{cpu_sleep} second..."
loop do
begin
mail = @imap_client.fetch_next_unseen(inbox: Protocols::Imap::BASE_INBOX)
if mail
handle_one(mail)
mail.seen!(imap_client: @imap_client, inbox: Protocols::Imap::BASE_INBOX)
end
rescue StandardError => e
$logger.error(e)
end
mail = @imap_client.fetch
handle_one(mail) if mail
sleep cpu_sleep
end

View File

@ -43,54 +43,6 @@ class Distributor
end
end
# Abstract action that check modo permissions & fetch email attr["uid"] from modo
class ModerationDistribute < Action
def handle_get_mail_to_distribute(list:, mail:, attributes:)
modo = Email.first(mailinglist: list, email: mail.from)
if !modo&.modo? && !modo&.op?
$logger.warn "SECU <#{mail.from}> failed to validate email on <#{list.email}>"
return nil
end
uid_mail = attributes["uid"]
if !uid_mail
$logger.warn "<#{mail.from}> did not specified email uid"
return nil
end
mail_to_distribute = @distributor.imap_client.fetch_first(
uid: uid_mail.to_i, inbox: Protocols::Imap::MODERATION_INBOX,
)
if !mail_to_distribute
$logger.warn "<#{mail.from}> tried to distribute unexisting uid #{uid_mail}"
return nil
end
mail_to_distribute
end
end
class ValidateDistribute < ModerationDistribute
def handle(list:, mail:, attributes:)
mail_to_distribute = handle_get_mail_to_distribute(list:, mail:, attributes:)
return if !mail_to_distribute
list.enabled_readers.each do |reader|
to_distrib = mail_to_distribute.to_redistribute(list:, dest: reader)
@distributor.distribute(to_distrib)
end
end
end
class RefuseDistribute < ModerationDistribute
def handle(list:, mail:, attributes:)
mail_to_distribute = handle_get_mail_to_distribute(list:, mail:, attributes:)
return if !mail_to_distribute
@distributor.imap_client.move mail: mail_to_distribute, to: Protocols::Imap::REFUSED_INBOX, inbox: Protocols::Imap::MODERATION_INBOX
end
end
class ListUsers < Action
end

View File

@ -4,8 +4,8 @@ class Distributor
class Subscribe < Action
FORBIDDEN_TEMPLATE = Actions.template("subscribe.forbidden")
SUCCESS_TEMPLATE = Actions.template("subscribe.success")
WAIT_USER_SUBSCRIBE_TEMPLATE = Actions.template("subscribe.wait_user")
WAIT_MODO_SUBSCRIBE_TEMPLATE = Actions.template("subscribe.wait_modo")
WAIT_USER_TEMPLATE = Actions.template("subscribe.wait_user")
WAIT_MODO_TEMPLATE = Actions.template("subscribe.wait_modo")
def handle(list:, mail:, attributes:)
register =
@ -17,25 +17,25 @@ class Distributor
end
if register
if !register.reader?
handle_wait_validation(list:, mail:, register:)
handle_wait_validation(list: list, mail: mail, register: register)
else
handle_subscribed(list:, mail:, register:)
handle_subscribed(list: list, mail: mail, register: register)
end
else
handle_403(list:, mail:)
handle_403(list: list, mail: mail)
end
end
def handle_wait_validation(list:, mail:, register:)
$logger.debug "Subscribe#handle_wait_validation on #{list.email} for #{mail.from}"
body = WAIT_USER_SUBSCRIBE_TEMPLATE.result binding
@distributor.distribute(mail.to_response(list:, mail:, body:))
$logger.debug "Subscribe#handle_wait_validation on #{list.email} for #{to.from}"
body = WAIT_USER_TEMPLATE.result binding
@distributor.distribute(mail.to_response(list: list, mail: mail, body: body))
modo = list.enabled_modos.first # TODO: send to all
body = WAIT_MODO_SUBSCRIBE_TEMPLATE.result binding
modo = list.enabled_modos.first
body = WAIT_MODO_TEMPLATE.result binding
@distributor.distribute(
Protocols::Mail.build(
subject: "ML #{list.name} requires validation for #{mail.from}", list:, to: modo.email, body:,
subject: "ML #{list.name} requires validaton for #{mail.from}", list: list, to: modo.email, body: body,
),
)
end
@ -44,13 +44,13 @@ class Distributor
$logger.debug "Subscribe#handle_subscribed on #{list.email} for #{mail.from}"
$logger.debug register.inspect
body = SUCCESS_TEMPLATE.result binding
@distributor.distribute(mail.to_response(list:, mail:, body:))
@distributor.distribute(mail.to_response(list: list, mail: mail, body: body))
end
def handle_403(list:, mail:)
$logger.debug "Subscribe#handle_403 on #{list.email} for #{mail.from}"
body = FORBIDDEN_TEMPLATE.result binding
@distributor.distribute(mail.to_response(list:, mail:, body:))
@distributor.distribute(mail.to_response(list: list, mail: mail, body: body))
end
end
@ -60,7 +60,7 @@ class Distributor
def handle(list:, mail:, attributes:)
Email.unregister!(mailinglist: list, email: mail.from)
body = SUCCESS_TEMPLATE.result binding
@distributor.distribute(mail.to_response(list:, mail:, body:))
@distributor.distribute(mail.to_response(list: list, mail: mail, body: body))
end
end
@ -69,55 +69,26 @@ class Distributor
def handle(list:, mail:, attributes:)
body = HELP_TEMPLATE.result binding
@distributor.distribute(mail.to_response(list:, mail:, body:))
@distributor.distribute(mail.to_response(list: list, mail: mail, body: body))
end
end
# This distribute the mail among the readers
class Distribute < Action
WAIT_MODO_DISTRIBUTE_TEMPLATE = Actions.template("distribute.wait_modo")
def handle(list:, mail:, attributes:)
user = Email.first(mailinglist: list, email: mail.from)
if list.registration?("autoregister")
if user.nil?
$logger.info "registering <#{mail.from}> to <#{list.email}>"
Email.register!(mailinglist: list, name: mail.from_name || mail.from.split("@").first,
email: mail.from).save
end
# ok let the mail pass the security check
elsif !user&.writer?
$logger.warn "invalid email writer for #{mail.from} on #{mail.to}"
return nil
end
if list.moderation?("restrictedwrite") && !user.modo? && !user.op?
handle_moderated_list(list:, mail:, attributes:)
else
list.enabled_readers.each do |reader|
to_distrib = mail.to_redistribute(list:, dest: reader)
@distributor.distribute(to_distrib)
if !list.enabled_writers.find { _1.email == mail.from }
if list.registration?("autoregister")
Email.register!(mailinglist: list, name: mail.from_name, email: mail.from).save
else
$logger.warn "invalid email writer for #{mail.from} on #{mail.to}"
return nil
end
end
end
# @param list [Mailinglist]
# @parma mail [Protocols::Mail]
def handle_moderated_list(list:, mail:, attributes:)
message_id = mail.message_id
@distributor.imap_client.moderate mail
uid = @distributor.imap_client.search_message_id(message_id, inbox: Protocols::Imap::MODERATION_INBOX)
modo = list.enabled_modos.first
# TODO: this is the most lazy code I wrote this week I think
# if multipart, get the boundary to add a new part in the top
boundary = mail.header("Content-Type").to_s.match(/boundary="([^"]+)"/)[1] rescue nil
body = WAIT_MODO_DISTRIBUTE_TEMPLATE.result binding
modo_mail = Protocols::Mail.build(
subject: "ML #{list.name} requires validation for [#{mail.subject}]",
list:, to: modo.email, body:,
replace_headers: mail.kept_headers,
)
@distributor.distribute(modo_mail)
list.enabled_readers.each do |reader|
to_distrib = mail.to_redistribute(list: list, dest: reader)
@distributor.distribute(to_distrib)
end
end
end

View File

@ -1,12 +0,0 @@
<% if boundary %>--<%= boundary %>
Content-Transfer-Encoding: quoted-printable
Content-Type: text/plain; charset="utf-8"; protected-headers="v1"
<% end %><%= mail.from %> Tried to send an email to <%= list.email %>
- You can validate with <mailto:<%= list.validate_distribute_email(uid) %>>
- You can refuse with <mailto:<%= list.refuse_distribute_email(uid) %>>
##################################
<%= mail.body %>

View File

@ -22,7 +22,7 @@ class Mailinglist < Sequel::Model($db)
super
end
def self.build(name:, aliasing: nil, suffix: nil, strategy: "closed|freewrite")
def self.build(name:, aliasing: nil, suffix: nil, strategy: "closed")
email = BASE_USER.dup
email << SUFFIX_SEPARATOR << suffix if suffix
aliasing = UUID.generate if !suffix && !aliasing
@ -71,8 +71,8 @@ class Mailinglist < Sequel::Model($db)
emails.filter { _1.permissions & Email::Permissions::WRITE != 0 }
end
def enabled_ops
emails.filter { _1.permissions & Email::Permissions::OP != 0 }
def enabled_admins
emails.filter { _1.permissions & Email::Permissions::ADMIN != 0 }
end
def enabled_modos
@ -104,15 +104,7 @@ class Mailinglist < Sequel::Model($db)
end
def owner_email
enabled_ops.first.email
end
def validate_distribute_email(uid)
"#{email}?subject=validate,uid=#{uid}"
end
def refuse_distribute_email(uid)
"#{email}?subject=refuse,uid=#{uid}"
"#{email}?subject=owner" # TODO: first user?
end
def set_permissions_email(*permissions_symbols, user_email: nil, permissions: nil)

View File

@ -4,170 +4,44 @@ require "net/imap"
#
# TODO: strengthen the network management to avoid connection loss.
class Protocols::Imap
MODERATION_INBOX = "INBOX/moderation".freeze
REFUSED_INBOX = "INBOX/refused".freeze
BASE_INBOX = "INBOX".freeze
ALL_INBOXES = [MODERATION_INBOX, REFUSED_INBOX, BASE_INBOX].freeze
def initialize
reset!
end
attr_reader :imap if $debug
def reset!
@imap = Net::IMAP.new(ENV["IMAP_HOST"], ENV["IMAP_PORT"], ssl: true)
@imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
ALL_INBOXES.each do
$logger.info "Try to create inbox #{_1}"
@imap.create(_1)
$logger.info "Inbox #{_1} created"
rescue StandardError => e
$logger.error(e.message)
end # TODO: properly do that
@imap.select("INBOX")
end
# Fetch the next incomming email as a Protocols::Mail.
#
# @return [Protocols::Mail]
def fetch_next_unseen(inbox:)
uid = search_unseen(inbox:).first
return nil if uid.nil?
fetch(uid:, inbox:)
end
# Fetch all UID that are not marked as SEEN yet in the inbox
#
# @param inbox [String]
# @return [Array(Integer)]
def search_unseen(inbox:)
goto_inbox(inbox)
@imap.uid_search(%w[NOT SEEN])
end
# Fetch the first UID of message matching with the given message_id.
# Will return nil if none are matching.
# It is possible for several messages to have the same message_id.
# In this case it is impossible to say which one will be returned.
# In practice, it is likely that the older message will be always returned.
#
# @param inbox [String]
# @return [Integer?]
def search_message_id(message_id, inbox:)
goto_inbox(inbox)
@imap.uid_search(["HEADER", "Message-ID", message_id]).first
end
# Open an given inbox. Do not multiply requests to the IMAP server.
#
# @param inbox [String] the name of the inbox repository
# @param readonly [TrueClass, FalseClass] if true, next operations won't modify the box (with move, store...)
#
# @example
#
# goto_inbox Protocols::Imap::BASE_INBOX
# goto_inbox Protocols::Imap::MODERATION_INBOX, readonly: true
def goto_inbox(inbox, readonly: false)
return if @current_inbox == inbox
$logger.debug "goto_inbox #{inbox}"
select_method = readonly ? :examine : :select
@imap.send(select_method, inbox)
def fetch
@imap.check
@current_inbox = inbox
end
id = @imap.search(%w[NOT SEEN]).first
return nil if id.nil?
# Fetch the first message by UID or ID if it exists.
#
# @param id [String?] optional, specify either id or uid
# @param uid [String?] optional, specify either id or uid
#
# @return [Protocols::Mail?]
def fetch_first(inbox:, uid: nil, id: nil)
goto_inbox(inbox)
fetch_method = id ? :fetch : :uid_fetch
imap_mail = @imap.send(
fetch_method,
id || uid,
[Protocols::ENVELOPE, Protocols::UID, Protocols::HEADERS, Protocols::BODYTEXT, Protocols::MSG_ID],
)&.first
imap_mail = @imap.fetch(
id, [Protocols::ENVELOPE, Protocols::UID, Protocols::HEADERS, Protocols::BODYTEXT, Protocols::MSG_ID],
).first
return nil if imap_mail.nil?
$logger.debug imap_mail.attr["RFC822.HEADER"]
Protocols::Mail.new(imap_mail:)
end
# Fetch one incoming mail as a Protocols::Mail.
# {see #fetch_first}
#
# @param id [String?]
# @param uid [String?]
#
# @example
#
# fetch(uid: 1, inbox: Protocols::Imap::BASE_INBOX)
#
# @return [Protocols::Mail]
def fetch(inbox:, id: nil, uid: nil)
raise "Need id OR uid to be set" if !id && !uid
uid = search_unseen(inbox:).first
mail = fetch_first(uid:, inbox:)
$logger.info "READ #{id || uid} #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
mail = Protocols::Mail.new(imap_mail:)
$logger.info "READ #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
mail
end
# Move an email to the another inbox.
#
# @param inbox [String?] inbox to take the email from. Do nothing if nil.
# @param mail [Protocols::Mail] mail to move
# @param to [String] inbox repository where the mail is placed
def move(mail:, to:, inbox:)
goto_inbox inbox
$logger.info "MOVED #{mail.uid} from #{inbox} to #{to}"
@imap.uid_move mail.uid, to
end
# Move an email to the moderated inbox.
#
# @param mail [Protocols::Mail]
def moderate(mail, inbox: BASE_INBOX)
move mail:, to: MODERATION_INBOX, inbox:
end
# # Remove an email from the moderated inbox.
# #
# # @param mail [Protocols::Mail]
# def unmoderate(mail)
# @imap.uid_move mail.uid, BASE_INBOX
# end
# Add the SEEN flag to a given email.
# This email will not be reprocessed again.
#
# @param imap_mail [Protocols::Mail]
def seen!(imap_mail, inbox:)
goto_inbox inbox
$logger.info "MARK #{imap_mail.attr['UID']} as SEEN"
@imap.uid_store imap_mail.attr["UID"], "+FLAGS", [Net::IMAP::SEEN]
end
# Mark all existing messages as SEEN. See {#seen!}
def see_all_messages!
$logger.info "MARK ALL NOT SEEN as SEEN"
goto_inbox BASE_INBOX
see_all!
goto_inbox MODERATION_INBOX
see_all!
@imap.search(%w[NOT SEEN]).each { @imap.store _1, "+FLAGS", [Net::IMAP::SEEN] }
end
# :nodoc:
private def see_all!
@imap.uid_search(%w[NOT SEEN]).each do
@imap.uid_store _1, "+FLAGS", [Net::IMAP::SEEN]
$logger.debug "MARK #{_1} as SEEN"
end
# Add the SEEN flag to a given email.
# This email will not be reprocessed again.
#
# @param imap_mail [Protocols::Mail]
def seen!(imap_mail)
$logger.info "MARK #{imap_mail.attr['UID']} as SEEN"
@imap.uid_store imap_mail.attr["UID"], "+FLAGS", [Net::IMAP::SEEN]
end
end

View File

@ -5,13 +5,13 @@ require "uuid"
# It handles multiple headers (having several times the same header key)
# but you must know that gmail seem to do not allow it.
class Protocols::Mail
attr_accessor :from_name, :from, :to_name, :to, :subject, :body, :uid, :message_id
attr_accessor :from_name, :from, :to_name, :to, :subject, :body
DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z".freeze
HEADERS_KEEP = %w[
Content-Type Content-Transfer-Encoding MIME-Version
Message-ID Subject From To Date In-Reply-To References
Content-Type Content-Transfer-Encoding User-Agent MIME-Version
].freeze
HEADERS_KEEP_WITH_ORIGINAL = HEADERS_KEEP + %w[Message-ID User-Agent From To Subject Subject Date In-Reply-To References]
USER_AGENT = "Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}) Mailinglist.rb/1.0".freeze
# @params imap_mail is any element outputed in the returned array of
@ -19,7 +19,6 @@ class Protocols::Mail
# - Protocols::ENVELOPE
# - Protocols::HEADERS
# - Protocols::BODYTEXT
# - Protocols::UID
def initialize(imap_mail: nil)
if imap_mail
@imap_mail = imap_mail
@ -33,13 +32,11 @@ class Protocols::Mail
@subject = envelope.subject
@body = body_text
@headers = Protocols::Mail.parse_rfc822_headers(body_head)
@headers.filter! { HEADERS_KEEP_WITH_ORIGINAL.include?(_1[0]) }
@headers.filter! { HEADERS_KEEP.include?(_1[0]) }
@seq = header("X-Sequence").to_i
@message_id = header("Message-id")
@uid = imap_mail.attr[Protocols::UID]
end
end
attr_reader :imap_mail if $debug
# Copy an existing mail.
# It will NOT work if the email do not contain a imap_mail.
@ -48,11 +45,10 @@ class Protocols::Mail
end
# Interface with Protocols::Mail
def seen!(imap_client:, inbox:)
def seen!(imap_client:)
if @imap_mail
imap_client.seen!(@imap_mail, inbox:)
imap_client.seen!(@imap_mail)
else
$logger.debug("Cannot mark #{uid} as seen because no @imap_mail")
false
end
end
@ -101,7 +97,8 @@ class Protocols::Mail
#
# @param list [Mailinglist]
# @param to [String] email of the target
def self.build(subject:, list:, to:, body:, replace_headers: [])
# @param cc [Array(String)] optional, emails (["mail@host", "Name <mail@host>", ...])
def self.build(subject:, list:, to:, body:, cc: [])
new = new()
new.replace_headers!(
["User-Agent", USER_AGENT],
@ -114,7 +111,7 @@ class Protocols::Mail
["Date", Time.now.strftime(DATE_FORMAT)],
["Message-Id", "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
)
new.replace_headers!(*replace_headers) if replace_headers
new.replace_headers!(["CC", cc.join(", ")]) if !cc.empty?
new.subject = subject
new.to = to
@ -177,11 +174,6 @@ class Protocols::Mail
@headers ||= []
end
# Generate a list of headers
def kept_headers
headers.filter { HEADERS_KEEP.include?(_1[0]) }
end
# Add new headers. Do not overwrite existing ones.
#
# @params new_headers is a [ [key, value], ...] array

View File

@ -7,34 +7,12 @@ class Protocols::Smtp
reset_smtp_client!
end
class DistributedCache < Array
def initialize(max_size: 100)
@max_size = max_size
super()
end
def <<(message_id)
prepend(message_id) if !message_id.nil?
slice! @max_size
end
end
def cache
(@cache ||= DistributedCache.new)
end
# @param mail [Protocols::Mail]
def distribute(mail)
if cache.include?(mail.message_id)
$logger.warn "Already distributed #{mail.message_id}"
return
end
$logger.info "SEND #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
smtp_raw = mail.to_smtp
$logger.debug smtp_raw.join("=====")
send_message_safe(*smtp_raw)
cache << mail.message_id
rescue StandardError => e
$logger.warn "FAILED #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
$logger.debug e