Update readme with more next steps

The cache was not working properly as it intercepted all the emails
except for the first receiver.

.gitignore

@@ -1,4 +1,6 @@
-.env
+.env*
 *.db
 *.sqlite*
 README.html
+.yardoc/
+bin/db_seed.local

.ruby-version

@@ -0,0 +1 @@
+3.1.2

Gemfile

@@ -3,28 +3,27 @@
 source "https://rubygems.org"
 
 # Mails proto
-gem "net-smtp", "~> 0.3.3"
 gem "net-imap", "~> 0.3.1"
+gem "net-smtp", "~> 0.3.3"
 
 # Mail template
 gem "erb", "~> 3.0"
 
 # Database
-gem "sqlite3", "~> 1.5"
 gem "sequel", "~> 5.62"
-gem "colorize", "~> 0.8.1"
+gem "sqlite3", "~> 1.5"
 
 # Web
-gem "sinatra", "~> 3.0"
 gem "puma", "~> 6.0"
+gem "sinatra", "~> 3.0"
 gem "slim", "~> 4.1"
 
 # Others
 gem "dotenv", "~> 2.8"
 gem "uuid", "~> 2.3"
-gem "semver", "~> 1.0"
 
 # Debug and stuff
 group :develop do
+  gem "mocha", "~> 2.0"
   gem "pry", "~> 0.14.1"
 end

Gemfile.lock

@@ -3,13 +3,14 @@ GEM
   specs:
     cgi (0.3.3)
     coderay (1.1.3)
-    colorize (0.8.1)
     dotenv (2.8.1)
     erb (3.0.0)
       cgi (>= 0.3.3)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     method_source (1.0.0)
+    mocha (2.0.2)
+      ruby2_keywords (>= 0.0.5)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-imap (0.3.1)
@@ -28,7 +29,6 @@ GEM
     rack-protection (3.0.3)
       rack
     ruby2_keywords (0.0.5)
-    semver (1.0.1)
     sequel (5.62.0)
     sinatra (3.0.3)
       mustermann (~> 3.0)
@@ -50,14 +50,13 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  colorize (~> 0.8.1)
   dotenv (~> 2.8)
   erb (~> 3.0)
+  mocha (~> 2.0)
   net-imap (~> 0.3.1)
   net-smtp (~> 0.3.3)
   pry (~> 0.14.1)
   puma (~> 6.0)
-  semver (~> 1.0)
   sequel (~> 5.62)
   sinatra (~> 3.0)
   slim (~> 4.1)

README.md

@@ -13,7 +13,7 @@
 - [x] Robust codebase for extensibility
 - [x] Robust network (handle network loss etc.)
 - [ ] Inbox cleanup
-- [ ] Moderation toolbox and email validation before distribution
+- [x] Moderation toolbox and email validation before distribution
 
 ## Deploy from sources
 
@@ -37,6 +37,30 @@ Copy and fill all the env variables in .env
     cp empty.env .env
     edit .env
 
+Here is the list of all the configuration available (sampled in empty.env):
+
+    SMTP_HOST      information to connect on the smtp
+    SMTP_PORT
+    SMTP_TLS       true or false
+    SMTP_USER
+    SMTP_PASSWORD
+    IMAP_HOST      only imap available
+    IMAP_PORT
+    IMAP_TLS
+    IMAP_USER
+    IMAP_PASSWORD
+    SENDER_HOST    the domain of the sender field, and the mailinglist addresses
+    SENDER         "true" to retrive the true sender, "list" to use the mailing list email
+                   else it is a static email address that will alway be used.
+    FROM           same as SENDER
+    DB_URL         sqlite://db/database.sqlite for instance
+    PORT           for WWW accees, not used yet
+    DEBUG          true/false
+    CPU_SLEEP      slow down the distributor
+    LOG_FILE       log stuff in the specified file
+
+
+
 ### Setup the database
 
     bin/db_migrate
@@ -53,11 +77,25 @@ After deploying it, there are some tools:
 
 - `bin/db_seed` to generate some data
 
+### Additional resources
+
+[How to manage mailinglists](doc/manage_mailinglists.md)
+
 ## Contributing
 
+### Next step
+
+- [ ] Handle multi-modo (currently admin operations send the mail to the first modo only)
+- [ ] Add web interface (for archives)
+- [ ] Auto cut citations and mailinglist signatures
+- [ ] Fetch history via email
+- [ ] Global admin system via email
+- [ ] Harder email security (check output server to verify authenticity)
+- [ ] Disable signature by mailinglist
+
 ### via Gitea
 
-1. Fork it (<https://git.sceptique.eu/Sceptique/mailinglistrb>)
+1. Fork it (<https://git.sceptique.eu/Sceptique/mailinglist.rb>)
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
@@ -65,9 +103,10 @@ After deploying it, there are some tools:
 
 ### via emails
 
-Checkout git-send-mail tutorial <https://git-send-email.io/>
+Checkout git-send-mail tutorial <https://git-send-email.io/>.
+The mailing list is <mailto:list+mailinglistrb@sceptique.eu>
 
-1. Clone it (<https://git.sceptique.eu/Sceptique/mailinglistrb>)
-2. Subscribe to the mailinglist to send your patch <mailto:list.mailinglistrb@sceptique.eu?subject=subscribe> (don't send your patch in this email)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Send your email to <mailto:list.mailinglistrb@sceptique.eu> after you are validated by modo
+1. Clone it (<https://git.sceptique.eu/Sceptique/mailinglist.rb>)
+2. Commit your changes (`git commit -am 'Add some feature'`)
+3. Register the mailinglist by sending a "subscribe" to it
+4. Send your patch to the mailing list

Rakefile

@@ -0,0 +1,40 @@
+require "minitest/test_task"
+
+Minitest::TestTask.create(:test) do |t|
+  t.libs << "test"
+  t.libs << "src"
+  t.warning = false
+  t.test_globs = ["test/**/test_*.rb"]
+end
+
+namespace "db" do
+  desc "Migrate the database to the lasted schema"
+  task "migrate" do
+    load "bin/db_migrate"
+  end
+
+  desc "Initialize database with dumby data"
+  task "dumb_seed" do
+    load "bin/db_seed"
+  end
+
+  desc "Initialize database with local data"
+  task "local_seed" do
+    load "bin/db_seed.local"
+  end
+
+  desc "Reset all tables, schema, data"
+  task "reset" do
+    require_relative "lib/app"
+    $db.tables.each { $db.drop_table _1 }
+  end
+
+  namespace "reset" do
+    task "stuff" do
+    end
+  end
+end
+
+task :default => :test
+
+task default: :test

bin/db_console

@@ -3,6 +3,8 @@
 $LOAD_PATH << File.join(Dir.pwd, "lib")
 
 require "app"
+require "protocols"
+require "models"
 require "pry"
 
 # mailinglist1 = Mailinglist.build(name: "FreeML", suffix: "free", strategy: "free").save

bin/db_mailinglist_create

@@ -5,6 +5,8 @@ $LOAD_PATH << File.join(Dir.pwd, "lib")
 require "app"
 require "optparse"
 require "uuid"
+require "protocols"
+require "models"
 
 options = {
   name: UUID.generate,
@@ -21,8 +23,17 @@ OptionParser.new do |opts|
   opts.on("-n=NAME", "--name=NAME", "Define the name of the option") do |v|
     options[:name] = v
   end
+
+  opts.on("-e=EMAIL", "--email=EMAIL", "Initialize the list with some emails, separated with ,") do |email|
+    options[:emails] ||= []
+    options[:emails] << email
+  end
 end.parse!
 
 options[:suffix] = options[:name].gsub(/[^a-zA-Z0-9]+/, '-')
 mailinglist = Mailinglist.build(name: options[:name], suffix: options[:suffix], strategy: options[:strategy]).save
 pp mailinglist
+
+options[:emails].each do |email|
+  pp Email.register!(name: email, email: email, mailinglist: mailinglist)
+end

bin/db_migrate

@@ -1,88 +1,7 @@
 #!/usr/bin/env ruby
 $LOAD_PATH << File.join(Dir.pwd, "lib")
 
-require "semver"
-require "colorize"
 require "app"
-require "semver"
 
-def current_version
-  $db[:meta].first[:version] rescue 0
-end
-
-def migrate(version, message = nil, &block)
-  puts "Check migration #{version}".on_blue
-  if current_version < version
-    puts "Migrate #{version}".blue
-    begin
-      yield
-      $db[:meta].update(version: version)
-      puts "Successfuly set version #{version}".green
-      puts message.green if message
-    rescue => err
-      puts err.message.on_red
-      exit 1
-    end
-  else
-    puts "Already migrated #{version}".yellow
-  end
-end
-
-migrate 1, "Initialized database" do
-  # this table only contains one entry
-  $db.create_table :meta do
-    primary_key :id
-    Int :version
-    String :code_version
-    String :code_date
-  end
-  $db[:meta].insert(version: 1)
-end rescue puts "already initialized".yellow
-
-migrate 2, "Initialize mailinglists" do
-  $db.create_table :mailinglists do
-    primary_key :id
-    column :name,                  String, null: false # Name to display
-    column :email,                 String, null: false # Mailing list email
-    column :count_handled,         Integer  # amount of email distributed
-    column :count_distributed,     Integer  # amount of email distributed
-    column :last_email,            DateTime # date of last distribution
-    column :strategy,              String, null: false # "free/validated/closed/..."
-    column :updated_at,            DateTime
-    column :created_at,            DateTime
-
-    index :name, unique: true
-    index :email, unique: true
-  end
-end rescue puts "mailinglists already exists".yellow
-
-migrate 3, "Initialize emails" do
-  $db.create_table :emails do
-    primary_key :id
-    foreign_key :mailinglist_id, :mailinglists, null: false #, on_delete: :cascade
-    column :name,           String, null: false, fixed: true, size: 50
-    column :email,          String, null: false, fixed: true, size: 254
-    column :permissions,    Integer # bitfield. 1=read 2=write 4=admin
-    column :admin,          TrueClass
-    column :updated_at,     DateTime
-    column :created_at,     DateTime
-
-    index %i[email], unique: false # list personnal subscriptions
-    index %i[email mailinglist_id], unique: true
-    index %i[mailinglist_id], unique: false # allow to search entries by mailinglist
-    index %i[mailinglist_id permissions], unique: false # find readers, writers, etc.
-  end
-end rescue puts "emails already exists".yellow
-
-puts "End migration".on_blue
-code_version = `git tag`.split("\n").map{ |str| Semver.new(str) }.sort.last
-code_date = if code_version
-  `git show #{code_version} --pretty="format:%as"`.split("\n").first
-else
-  Time.now
-end
-code_version ||= "v0.0.0"
-$db[:meta].update(code_version: code_version.to_s)
-$db[:meta].update(code_date: code_date.to_s)
-puts "Set code version to #{code_version}".green
-puts "Set code date to #{code_date}".green
+sequel_command = "bundle exec sequel -E -m db/migrations #{ENV['DB_URL']}"
+exec(sequel_command)

bin/db_seed

@@ -3,12 +3,12 @@
 $LOAD_PATH << File.join(Dir.pwd, "lib")
 
 require "app"
+require "protocols"
+require "models"
 
+mailinglist0 = Mailinglist.build(name: "AutoReg", suffix: "autoreg", strategy: "autoregister").save
 mailinglist1 = Mailinglist.build(name: "FreeML", suffix: "free", strategy: "free").save
 mailinglist2 = Mailinglist.build(name: "ValidatedML", suffix: "validated", strategy: "validated").save
 mailinglist3 = Mailinglist.build(name: "ClosedML", suffix: "closed", strategy: "closed").save
-pp mailinglist1, mailinglist2, mailinglist3
 
-# email1 = Email.register!(name: "AP", email: "arthur.poulet.hunk@sceptique.eu", mailinglist: mailinglist)
-# email2 = Email.register!(name: "AP2", email: "arthur.poulet.hunk2@sceptique.eu", mailinglist: mailinglist)
-# pp email1, email2
+pp mailinglist0, mailinglist1, mailinglist2, mailinglist3

bin/distributor

@@ -2,6 +2,17 @@
 $LOAD_PATH << File.join(Dir.pwd, "lib")
 
 require "app"
+require "protocols"
+require "models"
 require "distributor"
 
+Signal.trap("SIGINT") do
+  Thread.new do
+    $logger.info "SIGINT, closing the app peacefully"
+    $logger.close
+    exit 0
+  end.join
+end
+
+$logger.info "Starting app"
 Distributor.new.start(cpu_sleep: (ENV["CPU_SLEEP"] || 1).to_i)

bin/http

@@ -2,7 +2,10 @@
 $LOAD_PATH << File.join(Dir.pwd, "lib")
 
 require "app"
+require "protocols"
+require "models"
 require "sinatra"
+
 set :views, File.expand_path(File.join(settings.root + "/../lib/web/views"))
 
 require "web"

db/migrations/0001_create_mailinglists.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    create_table :mailinglists do
+      primary_key :id
+      column :name,                  String, null: false # Name to display
+      column :email,                 String, null: false # Mailing list email
+      column :count_handled,         Integer  # amount of email distributed
+      column :count_distributed,     Integer  # amount of email distributed
+      column :last_email,            DateTime # date of last distribution
+      column :strategy,              String, null: false # "free/validated/closed/..."
+      column :updated_at,            DateTime
+      column :created_at,            DateTime
+
+      index :name, unique: true
+      index :email, unique: true
+    end
+  end
+end

db/migrations/0002_create_emails.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    create_table :emails do
+      primary_key :id
+      foreign_key :mailinglist_id, :mailinglists, null: false #, on_delete: :cascade
+      column :name,           String, null: false, fixed: true, size: 50
+      column :email,          String, null: false, fixed: true, size: 254
+      column :permissions,    Integer # bitfield. 1=read 2=write 4=admin
+      column :updated_at,     DateTime
+      column :created_at,     DateTime
+
+      index %i[email], unique: false # list personnal subscriptions
+      index %i[email mailinglist_id], unique: true
+      index %i[mailinglist_id], unique: false # allow to search entries by mailinglist
+      index %i[mailinglist_id permissions], unique: false # find readers, writers, etc.
+    end
+  end
+end

doc/manage_mailinglists.md

@@ -0,0 +1,43 @@
+# Mailinglist management
+
+## What are strategies?
+
+There are 2 types of strategies: registration and moderation.
+Registration defines how new user can access the list.
+Moderation defines how user can write on the list.
+
+### Registration
+
+- autoregister: writing any email to the mailinglist will add the email of the user to the mailinglist
+- free: any email can register to the list by sending a "subscribe" email to the list
+- validated: like free but moderators needs to confirm the email with a "set-permissions" email. the user details are distributed to moderators first.
+- closed: nobody can register via email
+
+### Moderation
+
+- freewrite: anyone registered can distribute emails to the list
+- restrictedwrite: email needs approval by moderator before distribution. the email details are only distributed to moderators first.
+
+## How to interact with the list by email ?
+
+You need to send an email to the list with the subject being an action.
+Some action can take parameters in the subject or the body.
+
+Actions are:
+- set-permissions
+- validate
+- refuse
+- subscribe
+- unsubscribe
+- help
+- list-users
+- owner
+
+Parameters are key=value, which are separated with "," in the subject or "new line" in the body.
+
+### permissions parameters
+permissions: a permission mask (0=reader, 1=writer, 2=operator, 4=moderator)
+user-email: the email of the user
+
+### validate parameters
+uid: uid of the email

empty.env

@@ -11,8 +11,11 @@ IMAP_USER=
 IMAP_PASSWORD=
 
 SENDER_HOST=machin.fr
+SENDER=true
+FROM=true
 DB_URL="sqlite://dev.db"
 
 PORT=10081
 DEBUG=false
-CPU_CYCLE=1
+CPU_SLEEP=2
+LOG_FILE=/var/log/mailinglistrb.log

lib/app.rb

@@ -8,16 +8,3 @@ require "sequel"
 $db = Sequel.connect(ENV["DB_URL"])
 
 require_relative "logger"
-
-module Protocols
-  FULL     = "RFC822".freeze
-  HEADERS  = "RFC822.HEADER".freeze
-  BODYTEXT = "BODY[TEXT]".freeze
-  ENVELOPE = "ENVELOPE".freeze
-  UID      = "UID".freeze
-  SEQ      = "BODY[HEADER.FIELDS (X-SEQUENCE)]".freeze
-  MSG_ID   = "BODY[HEADER.FIELDS (MESSAGE-ID)]".freeze
-end
-
-require_relative "protocols"
-require_relative "models" rescue nil

lib/distributor.rb

@@ -1,11 +1,20 @@
 class Distributor
 
   class Attributes < Hash
+    # Look at the subject if there are attributes parts.
+    # If not, look at the body too.
+    # TODO: this part can be improved with smarter attr detection
     def self.parse(subject:, body:)
       new = Attributes.new
-      subject_added_attribute = false
-      subject.to_s.split(",").each { new.add_attribute!(_1); subject_added_attribute = true }
-      body.to_s.split("\r\n").each { new.add_attribute!(_1) } if body.include?("=") && subject_added_attribute == false
+      subject_parts = subject.to_s.split(",")
+      if !subject_parts.empty?
+        subject_parts.each do
+          new.add_attribute!(_1)
+        end
+      elsif body.include?("=")
+        body.to_s.split("\r\n").each { new.add_attribute!(_1) }
+      end
+
       new
     end
 
@@ -18,7 +27,7 @@ class Distributor
   def initialize
     @smtp_client = Protocols::Smtp.new
     @imap_client = Protocols::Imap.new
-    @imap_client.clean if ENV["HARD_RESET_NOT_SEEN_MESSAGE"] == "true"
+    @imap_client.see_all_messages! if ENV["HARD_RESET_NOT_SEEN_MESSAGE"] == "true"
     @handlers = {
       :default          => Actions::Distribute.new(distributor: self),
       "subscribe"       => Actions::Subscribe.new(distributor: self),
@@ -26,32 +35,45 @@ class Distributor
       "help"            => Actions::Help.new(distributor: self),
       "set-permissions" => Actions::SetPermissions.new(distributor: self),
       "list-users"      => Actions::ListUsers.new(distributor: self),
+      "validate"        => Actions::ValidateDistribute.new(distributor: self),
+      "refuse"          => Actions::RefuseDistribute.new(distributor: self),
     }
   end
+  attr_reader :imap_client, :smtp_client, :handlers # if $debug # TODO: more alias for imap/smtp clients?
 
+  # Make sure any incoming email is properly directed to the right action.
   def handle_one(mail)
     $logger.info "incoming email from #{mail.from} | #{mail.subject}"
     list = Mailinglist.search_mail(mail)
     if list
-      # TODO: create list from mail ?
       subject, subject_attributes = mail.subject.split(",", 2)
       attributes = Attributes.parse(subject: subject_attributes, body: mail.body)
       handler = @handlers[subject] || @handlers[:default]
-      $logger.info "#{handler.class}#handle on #{list.email} for #{mail.from}"
-      handler.handle(list: list, to: mail, attributes: attributes)
+      $logger.info "#{handler.class}#handle on <#{list.email}> for <#{mail.from}>"
+      handler.handle(list:, mail: mail, attributes:)
+    else
+      $logger.warn "list #{mail.to} do not exist (asked by #{mail.from})"
     end
-    mail.seen!(imap_client: @imap_client)
   end
 
+  # alias for {Protocols::Smtp#distribute}
   def distribute(*ary, **opt)
     @smtp_client.distribute(*ary, **opt)
   end
 
+  # Run the main loop that read all incoming emails.
   def start(cpu_sleep: 1)
-    puts "fetching new mail to distribute every #{cpu_sleep} second..."
+    $logger.info "fetching new mail to distribute every #{cpu_sleep} second..."
     loop do
-      mail = @imap_client.fetch
-      handle_one(mail) if mail
+      begin
+        mail = @imap_client.fetch_next_unseen(inbox: Protocols::Imap::BASE_INBOX)
+        if mail
+          handle_one(mail)
+          mail.seen!(imap_client: @imap_client, inbox: Protocols::Imap::BASE_INBOX)
+        end
+      rescue StandardError => e
+        $logger.error(e)
+      end
 
       sleep cpu_sleep
     end

lib/distributor/#subscribed.txt.erb#

@@ -1 +0,0 @@
-You have subscribed to <%= list.name %> <<%= list.email %>> 

lib/distributor/action.rb

@@ -11,7 +11,7 @@ class Distributor
         @distributor = distributor
       end
 
-      def handle(list:, to:, attributes:)
+      def handle(list:, mail:, attributes:)
         $logger.error "#{self.class} is not implemented yet"
       end
     end

lib/distributor/actions/admin.rb

@@ -4,42 +4,109 @@ class Distributor
     class SetPermissions < Action
       SET_PERMISSIONS_TEMPLATE = Actions.template("set_permissions.success")
 
-      def handle(list:, to:, attributes:)
+      def handle(list:, mail:, attributes:)
         return if attributes["user-email"].nil? # drop missing param
         return if attributes["permissions"].nil? # drop missing param
 
-        modo = Email.first(mailinglist: list, email: to.from)
+        modo = Email.first(mailinglist: list, email: mail.from)
         if !modo&.modo? && !modo&.op?
-          $logger.warn "SECU <#{to.from}> failed to set-permissions <#{list.email}> modo"
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions <#{list.email}> modo"
           return nil
         end
 
         user_email = attributes["user-email"]
         user = Email.first(mailinglist: list, email: user_email)
         if user.nil?
-          $logger.warn "SECU <#{to.from}> failed to set-permissions on non-existing email <#{user_email}>"
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions on non-existing email <#{user_email}>"
           return nil
         end
 
         if user.op? && !modo.op?
-          $logger.warn "SECU <#{to.from}> failed to set-permissions on op email <#{user_email}>"
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions on op email <#{user_email}>"
           return nil
         end
 
         permissions = attributes["permissions"].to_i
         if Email::Permissions.op?(permissions) && !modo.op?
-          $logger.warn "SECU <#{to.from}> failed to set op permissions on email <#{user_email}>"
+          $logger.warn "SECU <#{mail.from}> failed to set op permissions on email <#{user_email}>"
           return nil
         end
 
         user.permissions = permissions
         user.save
         body = SET_PERMISSIONS_TEMPLATE.result binding
-        @distributor.distribute(Protocols::Mail.build(subject: "ML #{list.name} subscription update", list: list, to: user.email, body: body))
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} subscription update", list:, to: user.email, body:
+          )
+        )
       end
     end
 
+    # Abstract action that check modo permissions & fetch email attr["uid"] from modo
+    class ModerationDistribute < Action
+      def handle_get_mail_to_distribute(list:, mail:, attributes:)
+        modo = Email.first(mailinglist: list, email: mail.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{mail.from}> failed to validate email on <#{list.email}>"
+          return nil
+        end
+
+        uid_mail = attributes["uid"]
+        if !uid_mail
+          $logger.warn "<#{mail.from}> did not specified email uid"
+          return nil
+        end
+
+        mail_to_distribute = @distributor.imap_client.fetch_first(
+          uid: uid_mail.to_i, inbox: Protocols::Imap::MODERATION_INBOX,
+        )
+        if !mail_to_distribute
+          $logger.warn "<#{mail.from}> tried to distribute unexisting uid #{uid_mail}"
+          return nil
+        end
+
+        mail_to_distribute
+      end
+    end
+
+    class ValidateDistribute < ModerationDistribute
+      def handle(list:, mail:, attributes:)
+        mail_to_distribute = handle_get_mail_to_distribute(list:, mail:, attributes:)
+        return if !mail_to_distribute
+
+        list.enabled_readers.each do |reader|
+          to_distrib = mail_to_distribute.to_redistribute(list:, dest: reader)
+          @distributor.distribute(to_distrib)
+        end
+      end
+    end
+
+    class RefuseDistribute < ModerationDistribute
+      def handle(list:, mail:, attributes:)
+        mail_to_distribute = handle_get_mail_to_distribute(list:, mail:, attributes:)
+        return if !mail_to_distribute
+
+        @distributor.imap_client.move mail: mail_to_distribute, to: Protocols::Imap::REFUSED_INBOX, inbox: Protocols::Imap::MODERATION_INBOX
+      end
+    end
+
+    LIST_USERS_TEMPLATE = Actions.template("list_users")
     class ListUsers < Action
+      def handle(list:, mail:, attributes:)
+        modo = Email.first(mailinglist: list, email: mail.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions <#{list.email}> modo"
+          return nil
+        end
+
+        body = LIST_USERS_TEMPLATE.result binding
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} user list", list:, to: modo.email, body:
+          )
+        )
+      end
     end
 
   end

lib/distributor/actions/user.rb

@@ -4,83 +4,122 @@ class Distributor
     class Subscribe < Action
       FORBIDDEN_TEMPLATE = Actions.template("subscribe.forbidden")
       SUCCESS_TEMPLATE = Actions.template("subscribe.success")
-      WAIT_USER_TEMPLATE = Actions.template("subscribe.wait_user")
-      WAIT_MODO_TEMPLATE = Actions.template("subscribe.wait_modo")
+      WAIT_USER_SUBSCRIBE_TEMPLATE = Actions.template("subscribe.wait_user")
+      WAIT_MODO_SUBSCRIBE_TEMPLATE = Actions.template("subscribe.wait_modo")
 
-      def handle(list:, to:, attributes:)
+      def handle(list:, mail:, attributes:)
         register =
           begin
-            Email.register!(mailinglist: list, name: to.from_name, email: to.from).save
-          rescue => err
-            $logger.error err.message
+            Email.register!(mailinglist: list, name: mail.from_name, email: mail.from).save
+          rescue StandardError => e
+            $logger.error e.message
             nil
           end
         if register
           if !register.reader?
-            handle_wait_validation(list: list, to: to, register: register)
+            handle_wait_validation(list:, mail:, register:)
           else
-            handle_subscribed(list: list, to: to, register: register)
+            handle_subscribed(list:, mail:, register:)
           end
         else
-          handle_403(list: list, to: to)
+          handle_403(list:, mail:)
         end
       end
 
-      def handle_wait_validation(list:, to:, register:)
-        $logger.debug "Subscribe#handle_wait_validation on #{list.email} for #{to.from}"
-        body = WAIT_USER_TEMPLATE.result binding
-        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+      def handle_wait_validation(list:, mail:, register:)
+        $logger.debug "Subscribe#handle_wait_validation on #{list.email} for #{mail.from}"
+        body = WAIT_USER_SUBSCRIBE_TEMPLATE.result binding
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
 
-        modo = list.enabled_modos.first
-        body = WAIT_MODO_TEMPLATE.result binding
-        @distributor.distribute(Protocols::Mail.build(subject: "ML #{list.name} requires validaton for #{to.from}", list: list, to: modo.email, body: body))
+        modo = list.enabled_modos.first # TODO: send to all
+        body = WAIT_MODO_SUBSCRIBE_TEMPLATE.result binding
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} requires validation for #{mail.from}", list:, to: modo.email, body:,
+          ),
+        )
       end
 
-      def handle_subscribed(list:, to:, register:)
-        $logger.debug "Subscribe#handle_subscribed on #{list.email} for #{to.from}"
+      def handle_subscribed(list:, mail:, register:)
+        $logger.debug "Subscribe#handle_subscribed on #{list.email} for #{mail.from}"
         $logger.debug register.inspect
         body = SUCCESS_TEMPLATE.result binding
-        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
       end
 
-      def handle_403(list:, to:)
-        $logger.debug "Subscribe#handle_403 on #{list.email} for #{to.from}"
+      def handle_403(list:, mail:)
+        $logger.debug "Subscribe#handle_403 on #{list.email} for #{mail.from}"
         body = FORBIDDEN_TEMPLATE.result binding
-        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
       end
     end
 
     class Unsubscribe < Action
       SUCCESS_TEMPLATE = Actions.template("unsubscribe.success")
 
-      def handle(list:, to:, attributes:)
-        Email.unregister!(mailinglist: list, email: to.from)
+      def handle(list:, mail:, attributes:)
+        Email.unregister!(mailinglist: list, email: mail.from)
         body = SUCCESS_TEMPLATE.result binding
-        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
       end
     end
 
     class Help < Action
       HELP_TEMPLATE = Actions.template("help")
-      def handle(list:, to:, attributes:)
+
+      def handle(list:, mail:, attributes:)
         body = HELP_TEMPLATE.result binding
-        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
       end
     end
 
     # This distribute the mail among the readers
     class Distribute < Action
-      def handle(list:, to:, attributes:)
-        if !list
-          warn "invalid email writer for #{mail.from} on #{mail.to}"
+      WAIT_MODO_DISTRIBUTE_TEMPLATE = Actions.template("distribute.wait_modo")
+
+      def handle(list:, mail:, attributes:)
+        user = Email.first(mailinglist: list, email: mail.from)
+        if list.registration?("autoregister")
+          if user.nil?
+            $logger.info "registering <#{mail.from}> to <#{list.email}>"
+            user = Email.register!(mailinglist: list, name: mail.from_name || mail.from.split("@").first,
+                                   email: mail.from).save
+          end
+          # ok let the mail pass the security check
+        elsif !user&.writer?
+          $logger.warn "invalid email writer for #{mail.from} on #{mail.to}"
           return nil
         end
 
-        list.enabled_readers.each do |reader|
-          to_distrib = to.to_redistribute(list: list, dest: reader)
-          @distributor.distribute(to_distrib)
+        if list.moderation?("restrictedwrite") && !user.modo? && !user.op?
+          handle_moderated_list(list:, mail:, attributes:)
+        else
+          list.enabled_readers.each do |reader|
+            to_distrib = mail.to_redistribute(list:, dest: reader)
+            @distributor.distribute(to_distrib)
+          end
         end
       end
+
+      # @param list [Mailinglist]
+      # @parma mail [Protocols::Mail]
+      def handle_moderated_list(list:, mail:, attributes:)
+        message_id = mail.message_id
+        @distributor.imap_client.moderate mail
+        uid = @distributor.imap_client.search_message_id(message_id, inbox: Protocols::Imap::MODERATION_INBOX)
+        modo = list.enabled_modos.first
+        # TODO: this is the most lazy code I wrote this week I think
+        # if multipart, get the boundary to add a new part in the top
+        boundary = mail.header("Content-Type").to_s.match(/boundary="([^"]+)"/)[1] rescue nil
+        wait_modo_distribute_template = Actions.template("distribute.wait_modo")
+        body = wait_modo_distribute_template.result binding
+        modo_mail = Protocols::Mail.build(
+          subject: "ML #{list.name} requires validation for [#{mail.subject}]",
+          list:, to: modo.email, body:,
+          replace_headers: mail.kept_headers,
+        )
+        @distributor.distribute(modo_mail)
+      end
     end
 
   end

lib/distributor/templates/distribute.wait_modo.txt.erb

@@ -0,0 +1,11 @@
+<% if boundary %>--<%= boundary %>
+Content-Type: text/plain; charset="utf-8"; protected-headers="v1"
+
+<% end %><%= mail.from %> Tried to send an email to <%= list.email %>
+
+- You can validate with <mailto:<%= list.validate_distribute_email(uid) %>>
+- You can refuse with <mailto:<%= list.refuse_distribute_email(uid) %>>
+
+##################################
+
+<%= mail.body %>

lib/distributor/templates/list_users.txt.erb

@@ -0,0 +1,5 @@
+List of the email subscribed to <%= list.name %> <<%= list.email %>>:
+
+<% list.emails.each do |email| %>
+  - <%= email.name %> <<%= email.email %>> (permissions=<%= email.permissions %>, created <%= email.created_at %>)
+<% end %>

lib/distributor/templates/subscribe.success.txt.erb

@@ -1,3 +1,3 @@
 You have subscribed to <%= list.name %> <<%= list.email %>>
 
-Your current permissions are <%= register.permissions.to_s(2).rjust(8, "0") %>
+Your current permissions are <%= register.permissions_words.join(', ') %>

lib/logger.rb

@@ -1,21 +1,31 @@
 require "logger"
 
-$logger = Logger.new(STDOUT)
+class MultiIO
+  def initialize(*targets)
+    @targets = targets
+  end
+
+  def <<(io)
+    @targets << io unless io == self
+    self
+  end
+
+  def write(*args)
+    @targets.each { _1.write(*args) }
+    self
+  end
+
+  def close
+    @targets.each(&:close)
+  end
+end
+
+log_output = MultiIO.new(STDOUT)
+if ENV["LOG_FILE"]
+  log_file = File.open(ENV["LOG_FILE"], "a")
+  log_output << log_file
+end
+
+$logger = Logger.new(log_output)
 $logger.level = Logger::INFO
 $logger.level = Logger::DEBUG if $debug
-
-def warn(*args, &block)
-  $logger.warn(*args, &block)
-end
-
-def puts(*args, &block)
-  $logger.info(*args, &block)
-end
-
-def error(*args, &block)
-  $logger.error(*args, &block)
-end
-
-def info(*args, &block)
-  $logger.info(*args, &block)
-end

lib/models/email.rb

@@ -32,7 +32,7 @@ class Email < Sequel::Model($db)
       none:     NONE,
     }
     def self.from_symbols(*syms)
-      syms.map { FROM_SYMBOLS[_1] }.sum rescue binding.pry
+      syms.map { FROM_SYMBOLS[_1] }.sum
     end
 
     def self.op?(perm)
@@ -83,16 +83,16 @@ class Email < Sequel::Model($db)
     permissions =
       if mailinglist.emails.count == 0
         Permissions::ALL
-      elsif mailinglist.strategy == "free"
+      elsif mailinglist.registration == "free"
         Permissions::READ | Permissions::WRITE
-      elsif mailinglist.strategy == "validated"
+      elsif mailinglist.registration == "validated"
         Permissions::NONE
-      elsif mailinglist.strategy == "closed"
+      elsif mailinglist.registration == "closed"
         $logger.warn "Forbidden register for #{mailinglist.id}"
         return nil # not valid
       end
 
-    $logger.debug { "Existing users: #{mailinglist.emails.map(&:email).join(", ")}" }
+    $logger.debug { "Existing users: #{mailinglist.emails.map(&:email).join(', ')}" }
     new(
       name: name,
       email: email,

lib/models/mailinglist.rb

@@ -1,13 +1,14 @@
 require "uuid"
 
 class Mailinglist < Sequel::Model($db)
-  SUFFIX_SEPARATOR = ENV["MAILINGLIST_SUFFIX_SEPARATOR"] || "."
+  SUFFIX_SEPARATOR = ENV["MAILINGLIST_SUFFIX_SEPARATOR"] || "+"
   BASE_USER = ENV["MAILINGLIST_BASE_USER"] || "mailinglist"
   HOST = ENV["MAILINGLIST_HOST"]
 
   STRATEGIES = {
-    registration: %w[free validated closed],
-  }
+    registration: %w[autoregister free validated closed],
+    moderation: %w[freewrite restrictedwrite],
+  }.freeze
 
   one_to_many :emails
 
@@ -21,8 +22,8 @@ class Mailinglist < Sequel::Model($db)
     super
   end
 
-  def self.build(name:, aliasing: nil, suffix: nil, strategy: "closed")
-    email = "#{BASE_USER}"
+  def self.build(name:, aliasing: nil, suffix: nil, strategy: "closed|freewrite")
+    email = BASE_USER.dup
     email << SUFFIX_SEPARATOR << suffix if suffix
     aliasing = UUID.generate if !suffix && !aliasing
     email << aliasing if aliasing
@@ -46,20 +47,36 @@ class Mailinglist < Sequel::Model($db)
     "\r\n\r\n---\r\nYou can unsubscribe to the mailinglist #{name} via: <mailto:#{unsubscribe_email}"
   end
 
+  def registration
+    strategy.split("|").filter { STRATEGIES[:registration].include?(_1) }
+  end
+
+  def registration?(find)
+    registration.include?(find)
+  end
+
+  def moderation
+    strategy.split("|").filter { STRATEGIES[:moderation].include?(_1) }
+  end
+
+  def moderation?(find)
+    moderation.include?(find)
+  end
+
   def enabled_readers
-    emails.filter{ _1.permissions & Email::Permissions::READ != 0 }
+    emails.filter { _1.permissions & Email::Permissions::READ != 0 }
   end
 
   def enabled_writers
-    emails.filter{ _1.permissions & Email::Permissions::WRITE != 0 }
+    emails.filter { _1.permissions & Email::Permissions::WRITE != 0 }
   end
 
-  def enabled_admins
-    emails.filter{ _1.permissions & Email::Permissions::ADMIN != 0 }
+  def enabled_ops
+    emails.filter { _1.permissions & Email::Permissions::OP != 0 }
   end
 
   def enabled_modos
-    emails.filter{ _1.permissions & Email::Permissions::MODO != 0 }
+    emails.filter { _1.permissions & Email::Permissions::MODO != 0 }
   end
 
   def actions_emails
@@ -69,7 +86,7 @@ class Mailinglist < Sequel::Model($db)
   ACTIONS_EMAILS = %i[
     help subscribe unsubscribe owner
     set_permissions list_users
-  ]
+  ].freeze
   def help_email
     "#{email}?subject=help"
   end
@@ -87,13 +104,19 @@ class Mailinglist < Sequel::Model($db)
   end
 
   def owner_email
-    "#{email}?subject=owner" # TODO: first user?
+    enabled_ops.first.email
+  end
+
+  def validate_distribute_email(uid)
+    "#{email}?subject=validate,uid=#{uid}"
+  end
+
+  def refuse_distribute_email(uid)
+    "#{email}?subject=refuse,uid=#{uid}"
   end
 
   def set_permissions_email(*permissions_symbols, user_email: nil, permissions: nil)
-    if permissions.nil?
-      permissions = Email::Permissions.from_symbols(*permissions_symbols)
-    end
+    permissions = Email::Permissions.from_symbols(*permissions_symbols) if permissions.nil?
     user_part = user_email ? ",user-email=#{user_email}" : ""
     permissions_part = ",permissions=#{permissions}"
     "#{email}?subject=set-permissions#{user_part}#{permissions_part}"
@@ -103,5 +126,4 @@ class Mailinglist < Sequel::Model($db)
     "#{email}?subject=list-users"
   end
 
-
 end

lib/protocols.rb

@@ -1,3 +1,20 @@
+# Protocols is a layer that help to interact with the ruby implementation of
+# IMAP, SMTP, and the mail standard.
+#
+# @example
+#     @smtp_client = Protocols::Smtp.new
+#     @imap_client = Protocols::Imap.new
+#     mail = Protocols::Mail.new(imap_mail: imap_mail)
+module Protocols
+  FULL     = "RFC822".freeze
+  HEADERS  = "RFC822.HEADER".freeze
+  BODYTEXT = "BODY[TEXT]".freeze
+  ENVELOPE = "ENVELOPE".freeze
+  UID      = "UID".freeze
+  SEQ      = "BODY[HEADER.FIELDS (X-SEQUENCE)]".freeze
+  MSG_ID   = "BODY[HEADER.FIELDS (MESSAGE-ID)]".freeze
+end
+
 require_relative "protocols/mail"
 require_relative "protocols/smtp"
 require_relative "protocols/imap"

lib/protocols/imap.rb

@@ -1,36 +1,183 @@
 require "net/imap"
 
+# Protocls::Imap allows to fetch emails.
+#
+# https://www.rfc-editor.org/rfc/rfc3501
+#
+# TODO: strengthen the network management to avoid connection loss.
 class Protocols::Imap
+  MODERATION_INBOX = "INBOX/moderation".freeze
+  REFUSED_INBOX = "INBOX/refused".freeze
+  BASE_INBOX = "INBOX".freeze
+  ALL_INBOXES = [MODERATION_INBOX, REFUSED_INBOX, BASE_INBOX].freeze
+
   def initialize
-    @imap = Net::IMAP.new(ENV["IMAP_HOST"], ENV["IMAP_PORT"], ssl: true)
+    reset!
+  end
+  attr_reader :imap if $debug
+
+  def reset!
+    @imap = Net::IMAP.new(ENV["IMAP_HOST"], port: ENV["IMAP_PORT"], ssl: true)
     @imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
-    @imap.select("INBOX")
+    ALL_INBOXES.each do
+      $logger.info "Try to create inbox #{_1}"
+      @imap.create(_1)
+      $logger.info "Inbox #{_1} created"
+    rescue StandardError => e
+      $logger.error(e.message)
+    end # TODO: properly do that
   end
 
-  def fetch
-    @imap.check
-    id = @imap.search(%w[NOT SEEN]).last
-    return nil if id.nil?
+  # Fetch the next incomming email as a Protocols::Mail.
+  #
+  # @return [Protocols::Mail]
+  def fetch_next_unseen(inbox:, max_tries: 2)
+    return $logger.error("fetch_next_unseen reached max_tries_limit") if max_tries == 0
 
-    imap_mail = @imap.fetch(
-      id, [Protocols::ENVELOPE, Protocols::UID, Protocols::HEADERS, Protocols::BODYTEXT, Protocols::MSG_ID],
-    ).first
+    begin
+      uid = search_unseen(inbox:).first
+    rescue IOError => e
+      $logger.warn e.message
+      reset!
+      return fetch_next_unseen(inbox:, max_tries: max_tries - 1)
+    end
+
+    return nil if uid.nil?
+    fetch(uid:, inbox:)
+  end
+
+  # Fetch all UID that are not marked as SEEN yet in the inbox
+  #
+  # @param inbox [String]
+  # @return [Array(Integer)]
+  def search_unseen(inbox:)
+    goto_inbox(inbox)
+    @imap.uid_search(%w[NOT SEEN])
+  end
+
+  # Fetch the first UID of message matching with the given message_id.
+  # Will return nil if none are matching.
+  # It is possible for several messages to have the same message_id.
+  # In this case it is impossible to say which one will be returned.
+  # In practice, it is likely that the older message will be always returned.
+  #
+  # @param inbox [String]
+  # @return [Integer?]
+  def search_message_id(message_id, inbox:)
+    goto_inbox(inbox)
+    @imap.uid_search(["HEADER", "Message-ID", message_id]).first
+  end
+
+  # Open an given inbox. Do not multiply requests to the IMAP server.
+  #
+  # @param inbox [String] the name of the inbox repository
+  # @param readonly [TrueClass, FalseClass] if true, next operations won't modify the box (with move, store...)
+  #
+  # @example
+  #
+  #   goto_inbox Protocols::Imap::BASE_INBOX
+  #   goto_inbox Protocols::Imap::MODERATION_INBOX, readonly: true
+  def goto_inbox(inbox, readonly: false)
+    return if @current_inbox == inbox
+
+    $logger.debug "goto_inbox #{inbox}"
+    select_method = readonly ? :examine : :select
+    @imap.send(select_method, inbox)
+    @imap.check
+    @current_inbox = inbox
+  end
+
+  # Fetch the first message by UID or ID if it exists.
+  #
+  # @param id [String?] optional, specify either id or uid
+  # @param uid [String?] optional, specify either id or uid
+  #
+  # @return [Protocols::Mail?]
+  def fetch_first(inbox:, uid: nil, id: nil)
+    goto_inbox(inbox)
+    fetch_method = id ? :fetch : :uid_fetch
+    imap_mail = @imap.send(
+      fetch_method,
+      id || uid,
+      [Protocols::ENVELOPE, Protocols::UID, Protocols::HEADERS, Protocols::BODYTEXT, Protocols::MSG_ID],
+    )&.first
     return nil if imap_mail.nil?
 
     $logger.debug imap_mail.attr["RFC822.HEADER"]
-    mail = Protocols::Mail.new(imap_mail: imap_mail)
-    puts "READ #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    Protocols::Mail.new(imap_mail:)
+  end
+
+  # Fetch one incoming mail as a Protocols::Mail.
+  # {see #fetch_first}
+  #
+  # @param id  [String?]
+  # @param uid [String?]
+  #
+  # @example
+  #
+  #   fetch(uid: 1, inbox: Protocols::Imap::BASE_INBOX)
+  #
+  # @return [Protocols::Mail]
+  def fetch(inbox:, id: nil, uid: nil)
+    raise "Need id OR uid to be set" if !id && !uid
+
+    uid = search_unseen(inbox:).first
+    mail = fetch_first(uid:, inbox:)
+
+    $logger.info "READ #{id || uid} #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
 
     mail
   end
 
-  def clean
-    puts "MARK ALL NOT SEEN as SEEN"
-    @imap.search(["NOT", "SEEN"]).each { @imap.store _1, "+FLAGS", [Net::IMAP::SEEN] }
+  # Move an email to the another inbox.
+  #
+  # @param inbox [String?]         inbox to take the email from. Do nothing if nil.
+  # @param mail  [Protocols::Mail] mail to move
+  # @param to    [String]          inbox repository where the mail is placed
+  def move(mail:, to:, inbox:)
+    goto_inbox inbox
+    $logger.info "MOVED #{mail.uid} from #{inbox} to #{to}"
+    @imap.uid_move mail.uid, to
   end
 
-  def seen!(imap_mail)
-    puts "MARK #{imap_mail.attr['UID']} as SEEN"
+  # Move an email to the moderated inbox.
+  #
+  # @param mail [Protocols::Mail]
+  def moderate(mail, inbox: BASE_INBOX)
+    move mail:, to: MODERATION_INBOX, inbox:
+  end
+
+  # # Remove an email from the moderated inbox.
+  # #
+  # # @param mail [Protocols::Mail]
+  # def unmoderate(mail)
+  #   @imap.uid_move mail.uid, BASE_INBOX
+  # end
+
+  # Add the SEEN flag to a given email.
+  # This email will not be reprocessed again.
+  #
+  # @param imap_mail [Protocols::Mail]
+  def seen!(imap_mail, inbox:)
+    goto_inbox inbox
+    $logger.info "MARK #{imap_mail.attr['UID']} as SEEN"
     @imap.uid_store imap_mail.attr["UID"], "+FLAGS", [Net::IMAP::SEEN]
   end
+
+  # Mark all existing messages as SEEN. See {#seen!}
+  def see_all_messages!
+    $logger.info "MARK ALL NOT SEEN as SEEN"
+    goto_inbox BASE_INBOX
+    see_all!
+    goto_inbox MODERATION_INBOX
+    see_all!
+  end
+
+  # :nodoc:
+  private def see_all!
+    @imap.uid_search(%w[NOT SEEN]).each do
+      @imap.uid_store _1, "+FLAGS", [Net::IMAP::SEEN]
+      $logger.debug "MARK #{_1} as SEEN"
+    end
+  end
 end

lib/protocols/mail.rb

@@ -1,57 +1,106 @@
 require "uuid"
 
+# Protocls::Mail provides an interface to read the content,
+# reply and redistribute the mail.
+# It handles multiple headers (having several times the same header key)
+# but you must know that gmail seem to do not allow it.
 class Protocols::Mail
-  attr_accessor :from_name, :from, :to_name, :to, :subject, :body
+  attr_accessor :from_name, :from, :to_name, :to, :subject, :body, :uid, :message_id
+
   DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z".freeze
+  HEADERS_KEEP = %w[
+    Content-Type Content-Transfer-Encoding MIME-Version
+  ].freeze
+  HEADERS_KEEP_WITH_ORIGINAL = HEADERS_KEEP + %w[Message-ID User-Agent From To Subject Subject Date In-Reply-To
+                                                 References]
+  USER_AGENT = "Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}) Mailinglist.rb/1.0".freeze
+  SENDER = ENV["SENDER"]
+  FROM = ENV["FROM"]
 
-  HEADERS_KEEP = %w[Message-ID Subject From To Date In-Reply-To References Content-Type Content-Transfer-Encoding User-Agent MIME-Version]
-  USER_AGENT = "Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}) Mailinglist.rb/1.0"
-
+  # @params imap_mail is any element outputed in the returned array of
+  #   Net::IMAP#fetch, and must contains some attributes:
+  #   - Protocols::ENVELOPE
+  #   - Protocols::HEADERS
+  #   - Protocols::BODYTEXT
+  #   - Protocols::UID
   def initialize(imap_mail: nil)
     if imap_mail
-      @imap_mail = imap_mail
-      envelope = imap_mail.attr[Protocols::ENVELOPE]
-      body_head = imap_mail.attr[Protocols::HEADERS]
-      body_text = imap_mail.attr[Protocols::BODYTEXT]
-      @from_name = envelope.from.first.name
-      @to_name   = envelope.to.first.name
-      @from      = "#{envelope.from.first.mailbox}@#{envelope.from.first.host}"
-      @to        = "#{envelope.to.first.mailbox}@#{envelope.to.first.host}"
-      @subject   = envelope.subject
-      @body      = body_text
-      @headers   = parse_rfc822_headers!(body_head)
-      @seq       = header("X-Sequence").to_i
+      @imap_mail  = imap_mail
+      envelope    = imap_mail.attr[Protocols::ENVELOPE]
+      body_head   = imap_mail.attr[Protocols::HEADERS]
+      body_text   = imap_mail.attr[Protocols::BODYTEXT]
+      @from_name  = envelope.from.first.name
+      @to_name    = envelope.to.first.name
+      @from       = "#{envelope.from.first.mailbox}@#{envelope.from.first.host}"
+      @to         = "#{envelope.to.first.mailbox}@#{envelope.to.first.host}"
+      @subject    = envelope.subject
+      @body       = body_text
+      @headers    = Protocols::Mail.parse_rfc822_headers(body_head)
+      @headers.filter! { HEADERS_KEEP_WITH_ORIGINAL.include?(_1[0]) }
+      @seq        = header("X-Sequence").to_i
       @message_id = header("Message-id")
+      @uid        = imap_mail.attr[Protocols::UID]
     end
   end
+  attr_reader :imap_mail if $debug
 
+  # Copy an existing mail.
+  # It will NOT work if the email do not contain a imap_mail.
   def clone
     Protocols::Mail.new(imap_mail: @imap_mail)
   end
 
-  def seen!(imap_client:)
+  # Interface with Protocols::Mail
+  def seen!(imap_client:, inbox:)
     if @imap_mail
-      imap_client.seen!(@imap_mail)
+      imap_client.seen!(@imap_mail, inbox:)
     else
+      $logger.debug("Cannot mark #{uid} as seen because no @imap_mail")
       false
     end
   end
 
+  # This method is a kind of hash.
+  def cache_id
+    "#{@message_id} #{@from} #{@to}"
+  end
+
   # Transform an received email into a mailing list email.
-  # Tested with evolution. Thunderbird bug at the moment.
+  # Tested with evolution and Kmail.
+  # Thunderbird bug at the moment which make it weird but it works too.
   #
-  # @param list   [Mailinglist]
+  # @param list [Mailinglist]
   # @param dest [Email]
   def to_redistribute(list:, dest:)
     new = clone
+    sender =
+      if SENDER == "true"
+        @from
+      elsif SENDER == "list"
+        list.email
+      else
+        SENDER
+      end
+
+    from_email =
+      if FROM == "true"
+        @from
+      elsif FROM == "list"
+        list.email
+      else
+        FROM
+      end
+
+    from = "\"#{@from_name}\" (via #{list.name}) <#{from_email}>"
+
     new.replace_headers!(
       ["User-Agent",       USER_AGENT],
       # require people to do not respond privatly
       ["Reply-to",         "#{list.name} <#{list.email}>"],
       ["Errors-To",        list.email],
       ["To",               dest.email],
-      ["From",             "\"#{@from_name}\" (via #{list.name}) <#{@from}>"],
-      ["Sender",           @from],
+      ["From",             from],
+      ["Sender",           sender ],
       ["Date",             Time.now.strftime(DATE_FORMAT)],
       ["List-Id",          "<#{list.email}>"],
       ["List-Post",        "<mailto:#{list.email}>"],
@@ -61,7 +110,7 @@ class Protocols::Mail
       ["List-Post",        "<mailto:#{list.post_email}>"],
       ["List-Owner",       "<mailto:#{list.owner_email}>"],
       ["In-Reply-To",      @message_id],
-      ["Precedence",       "list"],
+      %w[Precedence list],
       # ["Precedence",       "bulk"],
       ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
       ["X-Sequence",         (@seq + 1).to_s],
@@ -80,12 +129,10 @@ class Protocols::Mail
   #
   # @param list [Mailinglist]
   # @param to   [String]        email of the target
-  # @param cc   [Array(String)] optional, emails (["mail@host", "Name <mail@host>", ...])
-  def self.build(subject:, list:, to:, body:, cc: [])
+  def self.build(subject:, list:, to:, body:, replace_headers: [])
     new = new()
     new.replace_headers!(
       ["User-Agent",       USER_AGENT],
-      # require people to do not respond privatly
       ["Reply-to",         "#{list.name} <#{list.email}>"],
       ["Errors-To",        list.email],
       ["To",               to],
@@ -95,7 +142,7 @@ class Protocols::Mail
       ["Date",             Time.now.strftime(DATE_FORMAT)],
       ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
     )
-    new.replace_headers!(["CC", cc.join(", ")]) if !cc.empty?
+    new.replace_headers!(*replace_headers) if replace_headers
 
     new.subject = subject
     new.to   = to
@@ -108,8 +155,8 @@ class Protocols::Mail
   # Create a response email for an existing one.
   #
   # @param list [Mailinglist]
-  # @param to   [Protocols::Mail]
-  def to_response(list:, to:, body:)
+  # @param mail [Protocols::Mail]
+  def to_response(list:, mail:, body:)
     new = clone
     new.replace_headers!(
       ["User-Agent",       USER_AGENT],
@@ -123,13 +170,14 @@ class Protocols::Mail
       ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
     )
 
-    new.to   = to.from
+    new.to   = mail.from
     new.from = list.email
     new.body = "#{body}#{list.signature}"
     new
   end
 
-  def parse_rfc822_headers!(rfc822)
+  # :nodoc:
+  def self.parse_rfc822_headers(rfc822)
     headers = []
     rfc822.split("\r\n").each do |line|
       if line[0] == "\t"
@@ -139,41 +187,57 @@ class Protocols::Mail
         headers << [k, v.to_s.strip]
       end
     end
-    headers.filter! { HEADERS_KEEP.include?(_1[0]) }
+    headers
   end
 
+  # Get a header by key. Case insensitive. Only return the first occurence.
+  #
+  # @return [String?]
   def header(k)
     tuple = headers.find { _1[0].downcase == k.downcase }
     tuple && tuple[1]
   end
 
+  # Getter the all the parsed headers.
+  #
+  # @return [Array(Tuple(String, String))]
   def headers
     @headers ||= []
   end
 
+  # Generate a list of headers
+  def kept_headers
+    headers.filter { HEADERS_KEEP.include?(_1[0]) }
+  end
+
+  # Add new headers. Do not overwrite existing ones.
+  #
   # @params new_headers is a [ [key, value], ...] array
   def add_headers!(*new_headers)
     new_headers.each { headers << _1 }
   end
 
+  # Remove existing headers.
+  #
   # @param headers_keys is a [word, word, ...] array
   def delete_headers!(*headers_keys)
     headers_keys.each { delete_header! _1 }
   end
 
-  def delete_header!(header_key)
+  # :nodoc:
+  private def delete_header!(header_key)
     headers.delete_if { _1[0].downcase == header_key.downcase }
   end
 
+  # Remove conflicting headers and replace them with a new set.
+  # If the header do not exist yet it is simply added.
+  #
+  # @params update_headers is a [ [key, value], ...] array
   def replace_headers!(*update_headers)
     delete_headers!(*update_headers.map(&:first))
     add_headers!(*update_headers)
   end
 
-  private def smtp_headers
-    headers.map { _1.join(": ") }.join("\r\n")
-  end
-
   # TODO
   # List-Id: <mailto:>
   # List-Help: <mailto:>
@@ -185,8 +249,17 @@ class Protocols::Mail
   # Archived-At: <https://...>
   # List-Post: NO (posting not allowed on this list)
 
-  # https://www.ietf.org/rfc/rfc2369.txt
-  # https://www.rfc-editor.org/rfc/rfc5983
+  # :nodoc:
+  private def smtp_headers
+    headers.map { _1.join(": ") }.join("\r\n")
+  end
+
+  # Generate a Truple(String, String, String) that can be sent
+  # directly to the SMTP server using Net::SMTP or Protocols::SMTP.
+  #
+  # Notes:
+  # - https://www.ietf.org/rfc/rfc2369.txt
+  # - https://www.rfc-editor.org/rfc/rfc5983
   def to_smtp
     [
       "#{smtp_headers}\r\n\r\n#{@body}",

lib/protocols/smtp.rb

@@ -1,20 +1,46 @@
 require "net/smtp"
 
+# High level Interface to distribute Protocols::Mail objects.
+# It is robust against network loss.
 class Protocols::Smtp
   def initialize
     reset_smtp_client!
   end
 
+  class DistributedCache < Array
+    def initialize(max_size: 4096)
+      @max_size = max_size
+      super()
+    end
+
+    def <<(message_id)
+      prepend(message_id) if !message_id.nil?
+      slice! @max_size
+    end
+  end
+
+  def cache
+    (@cache ||= DistributedCache.new)
+  end
+
+  # @param mail [Protocols::Mail]
   def distribute(mail)
-    puts "SEND #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    if cache.include?(mail.cache_id)
+      $logger.warn "Already distributed #{mail.message_id}"
+      return
+    end
+
+    $logger.info "SEND #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
     smtp_raw = mail.to_smtp
     $logger.debug smtp_raw.join("=====")
     send_message_safe(*smtp_raw)
-  rescue => err
-    warn "FAILED #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
-    $logger.debug err
+    cache << mail.cache_id
+  rescue StandardError => e
+    $logger.warn "FAILED #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    $logger.debug e
   end
 
+  # :nodoc:
   private def reset_smtp_client!
     @smtp = Net::SMTP.new(
       ENV["SMTP_HOST"], ENV["SMTP_PORT"], tls: ENV["SMTP_TLS"] == "true",
@@ -25,17 +51,19 @@ class Protocols::Smtp
     )
   end
 
-  def send_message_safe(*raw, max_tries: 3)
-    if max_tries == 0
-      return $logger.error("send_message_safe reached max_tries_limit")
-    end
+  # :nodoc:
+  private def send_message_safe(*raw, max_tries: 3)
+    return $logger.error("send_message_safe reached max_tries_limit") if max_tries == 0
 
     begin
       @smtp.send_message(*raw)
-    rescue EOFError, Net::SMTPServerBusy => err
-      warn err.message
+    rescue EOFError, Net::SMTPServerBusy => e
+      $logger.warn e.message
       reset_smtp_client!
       send_message_safe(*raw, max_tries: max_tries - 1)
+    rescue => e
+      $logger.error e.full_message
+      reset_smtp_client!
     end
   end
 end

lib/semver.rb

@@ -1,44 +0,0 @@
-class Semver
-  def initialize(string)
-    @values = string.gsub(/^v(\d.+)/, '\1').split(".").map(&:to_i)
-  end
-
-  def [](index)
-    @values[index] || 0
-  end
-
-  def to_s
-    "v#{@values.join(".")}"
-  end
-
-  def <=>(left)
-    cmp = 0
-    @values.each_with_index do |value, index|
-      cmp = value - left[index]
-      break if cmp != 0
-    end
-    cmp
-  end
-
-  def >(left)
-    (self <=> left) > 0
-  end
-
-  def <(left)
-    (self <=> left) < 0
-  end
-
-  def ==(left)
-    (self <=> left) == 0
-  end
-
-  def !=(left)
-    (self <=> left) != 0
-  end
-end
-
-# pp Utils::Semver.new("1.0") < Utils::Semver.new("1.1")
-# pp Utils::Semver.new("0.1") < Utils::Semver.new("1.1")
-# pp Utils::Semver.new("1.1") < Utils::Semver.new("1.2")
-# pp Utils::Semver.new("1.1") < Utils::Semver.new("2.0")
-# pp Utils::Semver.new("12.0") > Utils::Semver.new("2.2")

test/protocols/test_mail.rb

@@ -0,0 +1,51 @@
+require "test_helper"
+require "protocols"
+
+module Protocols
+  class MailTest < Minitest::Test
+    def setup
+      @headers = <<~MAILEND
+        From: SenderH <sender@head.local>\r
+        To: DestH <dest@head.local>\r
+        Subject: -subject.head-\r
+        Date: #{DateTime.parse('2022-02-03T12:30:00Z').strftime(Protocols::Mail::DATE_FORMAT)}\r
+        Content-Type: plain/text\r
+        User-Agent: minitest\r
+      MAILEND
+      @imap_mail = OpenStruct.new(
+        attr: {
+          Protocols::ENVELOPE => OpenStruct.new(
+            from: [OpenStruct.new(mailbox: "sender", host: "local", name: "Sender")],
+            to: [OpenStruct.new(mailbox: "dest", host: "local", name: "Dest")],
+            subject: "-subject-",
+          ),
+          Protocols::HEADERS => @headers,
+          Protocols::BODYTEXT => "Content of the mail",
+        },
+      )
+      super
+    end
+
+    def test_parse_rfc822_headers
+      h1 = Mail.parse_rfc822_headers("Key: Value\r\n")
+      assert_equal 1, h1.size
+      h1.each { |tuple| assert_equal 2, tuple.size }
+      assert_equal "Key", h1.first.first
+      assert_equal "Value", h1.first.last
+
+      h2 = Mail.parse_rfc822_headers("Key: Value\r\n\tPartTwo\r\n")
+      assert_equal 1, h2.size
+      h2.each { |tuple| assert_equal 2, tuple.size }
+      assert_equal "Value\r\n\tPartTwo", h2.first.last
+    end
+
+    def test_init_with_imap_mail
+      mail = Mail.new(imap_mail: @imap_mail)
+      assert_equal "Sender", mail.from_name
+      assert_equal "sender@local", mail.from
+      assert_equal "Dest", mail.to_name
+      assert_equal "dest@local", mail.to
+      assert_equal "SenderH <sender@head.local>", mail.header("From")
+    end
+  end
+end

test/test_helper.rb

@@ -0,0 +1,3 @@
+require "pry"
+require "minitest"
+require "mocha/minitest"