Update readme with more next steps

The cache was not working properly as it intercepted all the emails
except for the first receiver.

.gitignore

@@ -1 +1,6 @@
-.env
+.env*
+*.db
+*.sqlite*
+README.html
+.yardoc/
+bin/db_seed.local

.ruby-version

@@ -0,0 +1 @@
+3.1.2

Gemfile

@@ -2,11 +2,28 @@
 
 source "https://rubygems.org"
 
-# gem "rails"
-
-gem "net-smtp", "~> 0.3.3"
+# Mails proto
 gem "net-imap", "~> 0.3.1"
+gem "net-smtp", "~> 0.3.3"
 
+# Mail template
+gem "erb", "~> 3.0"
+
+# Database
+gem "sequel", "~> 5.62"
+gem "sqlite3", "~> 1.5"
+
+# Web
+gem "puma", "~> 6.0"
+gem "sinatra", "~> 3.0"
+gem "slim", "~> 4.1"
+
+# Others
 gem "dotenv", "~> 2.8"
-
 gem "uuid", "~> 2.3"
+
+# Debug and stuff
+group :develop do
+  gem "mocha", "~> 2.0"
+  gem "pry", "~> 0.14.1"
+end

Gemfile.lock

@@ -1,16 +1,47 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    cgi (0.3.3)
+    coderay (1.1.3)
     dotenv (2.8.1)
+    erb (3.0.0)
+      cgi (>= 0.3.3)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
+    method_source (1.0.0)
+    mocha (2.0.2)
+      ruby2_keywords (>= 0.0.5)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
     net-imap (0.3.1)
       net-protocol
     net-protocol (0.1.3)
       timeout
     net-smtp (0.3.3)
       net-protocol
+    nio4r (2.5.8)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    puma (6.0.0)
+      nio4r (~> 2.0)
+    rack (2.2.4)
+    rack-protection (3.0.3)
+      rack
+    ruby2_keywords (0.0.5)
+    sequel (5.62.0)
+    sinatra (3.0.3)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.0.3)
+      tilt (~> 2.0)
+    slim (4.1.0)
+      temple (>= 0.7.6, < 0.9)
+      tilt (>= 2.0.6, < 2.1)
+    sqlite3 (1.5.3-x86_64-linux)
     systemu (2.6.5)
+    temple (0.8.2)
+    tilt (2.0.11)
     timeout (0.3.0)
     uuid (2.3.9)
       macaddr (~> 1.0)
@@ -20,8 +51,16 @@ PLATFORMS
 
 DEPENDENCIES
   dotenv (~> 2.8)
+  erb (~> 3.0)
+  mocha (~> 2.0)
   net-imap (~> 0.3.1)
   net-smtp (~> 0.3.3)
+  pry (~> 0.14.1)
+  puma (~> 6.0)
+  sequel (~> 5.62)
+  sinatra (~> 3.0)
+  slim (~> 4.1)
+  sqlite3 (~> 1.5)
   uuid (~> 2.3)
 
 BUNDLED WITH

README.md

@@ -0,0 +1,112 @@
+# mailinglist.rb
+
+## Features
+
+- [x] Receive new message and send it
+- [x] Have a database of mailinglists with name and users
+- [ ] Web interface to access archives
+- [x] ACL to restrict interactions with the system
+- [x] Install documentation
+- [x] Systemd service
+- [ ] AUR package
+- [x] Configuration for everything
+- [x] Robust codebase for extensibility
+- [x] Robust network (handle network loss etc.)
+- [ ] Inbox cleanup
+- [x] Moderation toolbox and email validation before distribution
+
+## Deploy from sources
+
+- Note there is a sample of systemd service you may use for your server in `/deploy/`
+
+### Download
+
+    git clone https://git.sceptique.eu/Sceptique/mailinglist.rb --depth 1
+    cd mailinglist.rb
+
+### Install dependencies
+
+System dependencies: `ruby 3.1.2`, `sqlite3`
+
+    bundle install
+
+### Configure
+
+Copy and fill all the env variables in .env
+
+    cp empty.env .env
+    edit .env
+
+Here is the list of all the configuration available (sampled in empty.env):
+
+    SMTP_HOST      information to connect on the smtp
+    SMTP_PORT
+    SMTP_TLS       true or false
+    SMTP_USER
+    SMTP_PASSWORD
+    IMAP_HOST      only imap available
+    IMAP_PORT
+    IMAP_TLS
+    IMAP_USER
+    IMAP_PASSWORD
+    SENDER_HOST    the domain of the sender field, and the mailinglist addresses
+    SENDER         "true" to retrive the true sender, "list" to use the mailing list email
+                   else it is a static email address that will alway be used.
+    FROM           same as SENDER
+    DB_URL         sqlite://db/database.sqlite for instance
+    PORT           for WWW accees, not used yet
+    DEBUG          true/false
+    CPU_SLEEP      slow down the distributor
+    LOG_FILE       log stuff in the specified file
+
+
+
+### Setup the database
+
+    bin/db_migrate
+
+### Start
+
+You may test it with
+
+    bin/distributor
+
+### Dev & play localy
+
+After deploying it, there are some tools:
+
+- `bin/db_seed` to generate some data
+
+### Additional resources
+
+[How to manage mailinglists](doc/manage_mailinglists.md)
+
+## Contributing
+
+### Next step
+
+- [ ] Handle multi-modo (currently admin operations send the mail to the first modo only)
+- [ ] Add web interface (for archives)
+- [ ] Auto cut citations and mailinglist signatures
+- [ ] Fetch history via email
+- [ ] Global admin system via email
+- [ ] Harder email security (check output server to verify authenticity)
+- [ ] Disable signature by mailinglist
+
+### via Gitea
+
+1. Fork it (<https://git.sceptique.eu/Sceptique/mailinglist.rb>)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+### via emails
+
+Checkout git-send-mail tutorial <https://git-send-email.io/>.
+The mailing list is <mailto:list+mailinglistrb@sceptique.eu>
+
+1. Clone it (<https://git.sceptique.eu/Sceptique/mailinglist.rb>)
+2. Commit your changes (`git commit -am 'Add some feature'`)
+3. Register the mailinglist by sending a "subscribe" to it
+4. Send your patch to the mailing list

Rakefile

@@ -0,0 +1,40 @@
+require "minitest/test_task"
+
+Minitest::TestTask.create(:test) do |t|
+  t.libs << "test"
+  t.libs << "src"
+  t.warning = false
+  t.test_globs = ["test/**/test_*.rb"]
+end
+
+namespace "db" do
+  desc "Migrate the database to the lasted schema"
+  task "migrate" do
+    load "bin/db_migrate"
+  end
+
+  desc "Initialize database with dumby data"
+  task "dumb_seed" do
+    load "bin/db_seed"
+  end
+
+  desc "Initialize database with local data"
+  task "local_seed" do
+    load "bin/db_seed.local"
+  end
+
+  desc "Reset all tables, schema, data"
+  task "reset" do
+    require_relative "lib/app"
+    $db.tables.each { $db.drop_table _1 }
+  end
+
+  namespace "reset" do
+    task "stuff" do
+    end
+  end
+end
+
+task :default => :test
+
+task default: :test

bin/db_console

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "protocols"
+require "models"
+require "pry"
+
+# mailinglist1 = Mailinglist.build(name: "FreeML", suffix: "free", strategy: "free").save
+# email1 = Email.register!(name: "AP", email: "arthur.poulet.hunk@sceptique.eu", mailinglist: mailinglist)
+binding.pry

bin/db_mailinglist_create

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "optparse"
+require "uuid"
+require "protocols"
+require "models"
+
+options = {
+  name: UUID.generate,
+  strategy: "free",
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: db_mailinglist_create [options]"
+
+  opts.on("-s=STRATEGY", "--strategy=STRATEGY", "free/validated/closed") do |v|
+    options[:strategy] = v
+  end
+
+  opts.on("-n=NAME", "--name=NAME", "Define the name of the option") do |v|
+    options[:name] = v
+  end
+
+  opts.on("-e=EMAIL", "--email=EMAIL", "Initialize the list with some emails, separated with ,") do |email|
+    options[:emails] ||= []
+    options[:emails] << email
+  end
+end.parse!
+
+options[:suffix] = options[:name].gsub(/[^a-zA-Z0-9]+/, '-')
+mailinglist = Mailinglist.build(name: options[:name], suffix: options[:suffix], strategy: options[:strategy]).save
+pp mailinglist
+
+options[:emails].each do |email|
+  pp Email.register!(name: email, email: email, mailinglist: mailinglist)
+end

bin/db_migrate

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+
+sequel_command = "bundle exec sequel -E -m db/migrations #{ENV['DB_URL']}"
+exec(sequel_command)

bin/db_seed

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "protocols"
+require "models"
+
+mailinglist0 = Mailinglist.build(name: "AutoReg", suffix: "autoreg", strategy: "autoregister").save
+mailinglist1 = Mailinglist.build(name: "FreeML", suffix: "free", strategy: "free").save
+mailinglist2 = Mailinglist.build(name: "ValidatedML", suffix: "validated", strategy: "validated").save
+mailinglist3 = Mailinglist.build(name: "ClosedML", suffix: "closed", strategy: "closed").save
+
+pp mailinglist0, mailinglist1, mailinglist2, mailinglist3

bin/distributor

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "protocols"
+require "models"
+require "distributor"
+
+Signal.trap("SIGINT") do
+  Thread.new do
+    $logger.info "SIGINT, closing the app peacefully"
+    $logger.close
+    exit 0
+  end.join
+end
+
+$logger.info "Starting app"
+Distributor.new.start(cpu_sleep: (ENV["CPU_SLEEP"] || 1).to_i)

bin/http

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "protocols"
+require "models"
+require "sinatra"
+
+set :views, File.expand_path(File.join(settings.root + "/../lib/web/views"))
+
+require "web"

db/migrations/0001_create_mailinglists.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    create_table :mailinglists do
+      primary_key :id
+      column :name,                  String, null: false # Name to display
+      column :email,                 String, null: false # Mailing list email
+      column :count_handled,         Integer  # amount of email distributed
+      column :count_distributed,     Integer  # amount of email distributed
+      column :last_email,            DateTime # date of last distribution
+      column :strategy,              String, null: false # "free/validated/closed/..."
+      column :updated_at,            DateTime
+      column :created_at,            DateTime
+
+      index :name, unique: true
+      index :email, unique: true
+    end
+  end
+end

db/migrations/0002_create_emails.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    create_table :emails do
+      primary_key :id
+      foreign_key :mailinglist_id, :mailinglists, null: false #, on_delete: :cascade
+      column :name,           String, null: false, fixed: true, size: 50
+      column :email,          String, null: false, fixed: true, size: 254
+      column :permissions,    Integer # bitfield. 1=read 2=write 4=admin
+      column :updated_at,     DateTime
+      column :created_at,     DateTime
+
+      index %i[email], unique: false # list personnal subscriptions
+      index %i[email mailinglist_id], unique: true
+      index %i[mailinglist_id], unique: false # allow to search entries by mailinglist
+      index %i[mailinglist_id permissions], unique: false # find readers, writers, etc.
+    end
+  end
+end

deploy/mailinglistrb.service

@@ -0,0 +1,35 @@
+[Unit]
+Description=Mailinglist.rb
+Documentation=https://git.sceptique.eu/Sceptique/mailinglist.rb
+
+[Service]
+ExecStart=/opt/mailinglistrb/bin/distributor
+Restart=on-failure
+RestartSec=3
+User=mailinglistrb
+Group=mailinglistrb
+WorkingDirectory=/opt/mailinglistrb
+Environment=/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl
+
+SystemCallArchitectures=native
+CapabilityBoundingSet=
+NoNewPrivileges=true
+PrivateDevices=true
+RemoveIPC=true
+LockPersonality=true
+
+ProtectControlGroups=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectKernelLogs=true
+ProtectClock=true
+ProtectHostname=true
+ProtectProc=noaccess
+
+RestrictRealtime=true
+RestrictSUIDSGID=true
+RestrictNamespaces=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+
+[Install]
+WantedBy=default.target

doc/manage_mailinglists.md

@@ -0,0 +1,43 @@
+# Mailinglist management
+
+## What are strategies?
+
+There are 2 types of strategies: registration and moderation.
+Registration defines how new user can access the list.
+Moderation defines how user can write on the list.
+
+### Registration
+
+- autoregister: writing any email to the mailinglist will add the email of the user to the mailinglist
+- free: any email can register to the list by sending a "subscribe" email to the list
+- validated: like free but moderators needs to confirm the email with a "set-permissions" email. the user details are distributed to moderators first.
+- closed: nobody can register via email
+
+### Moderation
+
+- freewrite: anyone registered can distribute emails to the list
+- restrictedwrite: email needs approval by moderator before distribution. the email details are only distributed to moderators first.
+
+## How to interact with the list by email ?
+
+You need to send an email to the list with the subject being an action.
+Some action can take parameters in the subject or the body.
+
+Actions are:
+- set-permissions
+- validate
+- refuse
+- subscribe
+- unsubscribe
+- help
+- list-users
+- owner
+
+Parameters are key=value, which are separated with "," in the subject or "new line" in the body.
+
+### permissions parameters
+permissions: a permission mask (0=reader, 1=writer, 2=operator, 4=moderator)
+user-email: the email of the user
+
+### validate parameters
+uid: uid of the email

empty.env

@@ -0,0 +1,21 @@
+SMTP_HOST=
+SMTP_PORT=465
+SMTP_TLS=true
+SMTP_USER=
+SMTP_PASSWORD=
+
+IMAP_HOST=
+IMAP_PORT=993
+IMAP_TLS=true
+IMAP_USER=
+IMAP_PASSWORD=
+
+SENDER_HOST=machin.fr
+SENDER=true
+FROM=true
+DB_URL="sqlite://dev.db"
+
+PORT=10081
+DEBUG=false
+CPU_SLEEP=2
+LOG_FILE=/var/log/mailinglistrb.log

lib/app.rb

@@ -0,0 +1,10 @@
+require "dotenv"
+Dotenv.load!
+
+require "pry"
+$debug = ENV["DEBUG"] == "true"
+
+require "sequel"
+$db = Sequel.connect(ENV["DB_URL"])
+
+require_relative "logger"

lib/distributor.rb

@@ -0,0 +1,83 @@
+class Distributor
+
+  class Attributes < Hash
+    # Look at the subject if there are attributes parts.
+    # If not, look at the body too.
+    # TODO: this part can be improved with smarter attr detection
+    def self.parse(subject:, body:)
+      new = Attributes.new
+      subject_parts = subject.to_s.split(",")
+      if !subject_parts.empty?
+        subject_parts.each do
+          new.add_attribute!(_1)
+        end
+      elsif body.include?("=")
+        body.to_s.split("\r\n").each { new.add_attribute!(_1) }
+      end
+
+      new
+    end
+
+    def add_attribute!(key_eq_value)
+      k, v = key_eq_value.split("=", 2)
+      self[k.strip] = v.strip if k && v
+    end
+  end
+
+  def initialize
+    @smtp_client = Protocols::Smtp.new
+    @imap_client = Protocols::Imap.new
+    @imap_client.see_all_messages! if ENV["HARD_RESET_NOT_SEEN_MESSAGE"] == "true"
+    @handlers = {
+      :default          => Actions::Distribute.new(distributor: self),
+      "subscribe"       => Actions::Subscribe.new(distributor: self),
+      "unsubscribe"     => Actions::Unsubscribe.new(distributor: self),
+      "help"            => Actions::Help.new(distributor: self),
+      "set-permissions" => Actions::SetPermissions.new(distributor: self),
+      "list-users"      => Actions::ListUsers.new(distributor: self),
+      "validate"        => Actions::ValidateDistribute.new(distributor: self),
+      "refuse"          => Actions::RefuseDistribute.new(distributor: self),
+    }
+  end
+  attr_reader :imap_client, :smtp_client, :handlers # if $debug # TODO: more alias for imap/smtp clients?
+
+  # Make sure any incoming email is properly directed to the right action.
+  def handle_one(mail)
+    $logger.info "incoming email from #{mail.from} | #{mail.subject}"
+    list = Mailinglist.search_mail(mail)
+    if list
+      subject, subject_attributes = mail.subject.split(",", 2)
+      attributes = Attributes.parse(subject: subject_attributes, body: mail.body)
+      handler = @handlers[subject] || @handlers[:default]
+      $logger.info "#{handler.class}#handle on <#{list.email}> for <#{mail.from}>"
+      handler.handle(list:, mail: mail, attributes:)
+    else
+      $logger.warn "list #{mail.to} do not exist (asked by #{mail.from})"
+    end
+  end
+
+  # alias for {Protocols::Smtp#distribute}
+  def distribute(*ary, **opt)
+    @smtp_client.distribute(*ary, **opt)
+  end
+
+  # Run the main loop that read all incoming emails.
+  def start(cpu_sleep: 1)
+    $logger.info "fetching new mail to distribute every #{cpu_sleep} second..."
+    loop do
+      begin
+        mail = @imap_client.fetch_next_unseen(inbox: Protocols::Imap::BASE_INBOX)
+        if mail
+          handle_one(mail)
+          mail.seen!(imap_client: @imap_client, inbox: Protocols::Imap::BASE_INBOX)
+        end
+      rescue StandardError => e
+        $logger.error(e)
+      end
+
+      sleep cpu_sleep
+    end
+  end
+
+  require_relative "distributor/action"
+end

lib/distributor/action.rb

@@ -0,0 +1,26 @@
+require "erb"
+
+class Distributor
+  module Actions
+
+    # Abstract
+    class Action
+      attr_reader :distributor
+
+      def initialize(distributor:)
+        @distributor = distributor
+      end
+
+      def handle(list:, mail:, attributes:)
+        $logger.error "#{self.class} is not implemented yet"
+      end
+    end
+
+    def self.template(name)
+      ERB.new(File.read(File.join(__dir__, "templates", "#{name}.txt.erb")))
+    end
+  end
+end
+
+require_relative "actions/admin"
+require_relative "actions/user"

lib/distributor/actions/admin.rb

@@ -0,0 +1,113 @@
+class Distributor
+  module Actions
+
+    class SetPermissions < Action
+      SET_PERMISSIONS_TEMPLATE = Actions.template("set_permissions.success")
+
+      def handle(list:, mail:, attributes:)
+        return if attributes["user-email"].nil? # drop missing param
+        return if attributes["permissions"].nil? # drop missing param
+
+        modo = Email.first(mailinglist: list, email: mail.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions <#{list.email}> modo"
+          return nil
+        end
+
+        user_email = attributes["user-email"]
+        user = Email.first(mailinglist: list, email: user_email)
+        if user.nil?
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions on non-existing email <#{user_email}>"
+          return nil
+        end
+
+        if user.op? && !modo.op?
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions on op email <#{user_email}>"
+          return nil
+        end
+
+        permissions = attributes["permissions"].to_i
+        if Email::Permissions.op?(permissions) && !modo.op?
+          $logger.warn "SECU <#{mail.from}> failed to set op permissions on email <#{user_email}>"
+          return nil
+        end
+
+        user.permissions = permissions
+        user.save
+        body = SET_PERMISSIONS_TEMPLATE.result binding
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} subscription update", list:, to: user.email, body:
+          )
+        )
+      end
+    end
+
+    # Abstract action that check modo permissions & fetch email attr["uid"] from modo
+    class ModerationDistribute < Action
+      def handle_get_mail_to_distribute(list:, mail:, attributes:)
+        modo = Email.first(mailinglist: list, email: mail.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{mail.from}> failed to validate email on <#{list.email}>"
+          return nil
+        end
+
+        uid_mail = attributes["uid"]
+        if !uid_mail
+          $logger.warn "<#{mail.from}> did not specified email uid"
+          return nil
+        end
+
+        mail_to_distribute = @distributor.imap_client.fetch_first(
+          uid: uid_mail.to_i, inbox: Protocols::Imap::MODERATION_INBOX,
+        )
+        if !mail_to_distribute
+          $logger.warn "<#{mail.from}> tried to distribute unexisting uid #{uid_mail}"
+          return nil
+        end
+
+        mail_to_distribute
+      end
+    end
+
+    class ValidateDistribute < ModerationDistribute
+      def handle(list:, mail:, attributes:)
+        mail_to_distribute = handle_get_mail_to_distribute(list:, mail:, attributes:)
+        return if !mail_to_distribute
+
+        list.enabled_readers.each do |reader|
+          to_distrib = mail_to_distribute.to_redistribute(list:, dest: reader)
+          @distributor.distribute(to_distrib)
+        end
+      end
+    end
+
+    class RefuseDistribute < ModerationDistribute
+      def handle(list:, mail:, attributes:)
+        mail_to_distribute = handle_get_mail_to_distribute(list:, mail:, attributes:)
+        return if !mail_to_distribute
+
+        @distributor.imap_client.move mail: mail_to_distribute, to: Protocols::Imap::REFUSED_INBOX, inbox: Protocols::Imap::MODERATION_INBOX
+      end
+    end
+
+    LIST_USERS_TEMPLATE = Actions.template("list_users")
+    class ListUsers < Action
+      def handle(list:, mail:, attributes:)
+        modo = Email.first(mailinglist: list, email: mail.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions <#{list.email}> modo"
+          return nil
+        end
+
+        body = LIST_USERS_TEMPLATE.result binding
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} user list", list:, to: modo.email, body:
+          )
+        )
+      end
+    end
+
+  end
+end

lib/distributor/actions/user.rb

@@ -0,0 +1,126 @@
+class Distributor
+  module Actions
+
+    class Subscribe < Action
+      FORBIDDEN_TEMPLATE = Actions.template("subscribe.forbidden")
+      SUCCESS_TEMPLATE = Actions.template("subscribe.success")
+      WAIT_USER_SUBSCRIBE_TEMPLATE = Actions.template("subscribe.wait_user")
+      WAIT_MODO_SUBSCRIBE_TEMPLATE = Actions.template("subscribe.wait_modo")
+
+      def handle(list:, mail:, attributes:)
+        register =
+          begin
+            Email.register!(mailinglist: list, name: mail.from_name, email: mail.from).save
+          rescue StandardError => e
+            $logger.error e.message
+            nil
+          end
+        if register
+          if !register.reader?
+            handle_wait_validation(list:, mail:, register:)
+          else
+            handle_subscribed(list:, mail:, register:)
+          end
+        else
+          handle_403(list:, mail:)
+        end
+      end
+
+      def handle_wait_validation(list:, mail:, register:)
+        $logger.debug "Subscribe#handle_wait_validation on #{list.email} for #{mail.from}"
+        body = WAIT_USER_SUBSCRIBE_TEMPLATE.result binding
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
+
+        modo = list.enabled_modos.first # TODO: send to all
+        body = WAIT_MODO_SUBSCRIBE_TEMPLATE.result binding
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} requires validation for #{mail.from}", list:, to: modo.email, body:,
+          ),
+        )
+      end
+
+      def handle_subscribed(list:, mail:, register:)
+        $logger.debug "Subscribe#handle_subscribed on #{list.email} for #{mail.from}"
+        $logger.debug register.inspect
+        body = SUCCESS_TEMPLATE.result binding
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
+      end
+
+      def handle_403(list:, mail:)
+        $logger.debug "Subscribe#handle_403 on #{list.email} for #{mail.from}"
+        body = FORBIDDEN_TEMPLATE.result binding
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
+      end
+    end
+
+    class Unsubscribe < Action
+      SUCCESS_TEMPLATE = Actions.template("unsubscribe.success")
+
+      def handle(list:, mail:, attributes:)
+        Email.unregister!(mailinglist: list, email: mail.from)
+        body = SUCCESS_TEMPLATE.result binding
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
+      end
+    end
+
+    class Help < Action
+      HELP_TEMPLATE = Actions.template("help")
+
+      def handle(list:, mail:, attributes:)
+        body = HELP_TEMPLATE.result binding
+        @distributor.distribute(mail.to_response(list:, mail:, body:))
+      end
+    end
+
+    # This distribute the mail among the readers
+    class Distribute < Action
+      WAIT_MODO_DISTRIBUTE_TEMPLATE = Actions.template("distribute.wait_modo")
+
+      def handle(list:, mail:, attributes:)
+        user = Email.first(mailinglist: list, email: mail.from)
+        if list.registration?("autoregister")
+          if user.nil?
+            $logger.info "registering <#{mail.from}> to <#{list.email}>"
+            user = Email.register!(mailinglist: list, name: mail.from_name || mail.from.split("@").first,
+                                   email: mail.from).save
+          end
+          # ok let the mail pass the security check
+        elsif !user&.writer?
+          $logger.warn "invalid email writer for #{mail.from} on #{mail.to}"
+          return nil
+        end
+
+        if list.moderation?("restrictedwrite") && !user.modo? && !user.op?
+          handle_moderated_list(list:, mail:, attributes:)
+        else
+          list.enabled_readers.each do |reader|
+            to_distrib = mail.to_redistribute(list:, dest: reader)
+            @distributor.distribute(to_distrib)
+          end
+        end
+      end
+
+      # @param list [Mailinglist]
+      # @parma mail [Protocols::Mail]
+      def handle_moderated_list(list:, mail:, attributes:)
+        message_id = mail.message_id
+        @distributor.imap_client.moderate mail
+        uid = @distributor.imap_client.search_message_id(message_id, inbox: Protocols::Imap::MODERATION_INBOX)
+        modo = list.enabled_modos.first
+        # TODO: this is the most lazy code I wrote this week I think
+        # if multipart, get the boundary to add a new part in the top
+        boundary = mail.header("Content-Type").to_s.match(/boundary="([^"]+)"/)[1] rescue nil
+        wait_modo_distribute_template = Actions.template("distribute.wait_modo")
+        body = wait_modo_distribute_template.result binding
+        modo_mail = Protocols::Mail.build(
+          subject: "ML #{list.name} requires validation for [#{mail.subject}]",
+          list:, to: modo.email, body:,
+          replace_headers: mail.kept_headers,
+        )
+        @distributor.distribute(modo_mail)
+      end
+    end
+
+  end
+end

lib/distributor/templates/distribute.wait_modo.txt.erb

@@ -0,0 +1,11 @@
+<% if boundary %>--<%= boundary %>
+Content-Type: text/plain; charset="utf-8"; protected-headers="v1"
+
+<% end %><%= mail.from %> Tried to send an email to <%= list.email %>
+
+- You can validate with <mailto:<%= list.validate_distribute_email(uid) %>>
+- You can refuse with <mailto:<%= list.refuse_distribute_email(uid) %>>
+
+##################################
+
+<%= mail.body %>

lib/distributor/templates/help.txt.erb

@@ -0,0 +1,25 @@
+# Mailing list: <%= list.name %> <<%= list.email %>>
+
+## Actions
+
+<% list.actions_emails.each do |action, action_email| %>
+- <%= action %> <mailto:<%= action_email %>><% end %>
+
+## User guide
+
+Most action are done via subject.
+Specify the action keyword in the subject or the body.
+You may add subjects arguments separated with <,>, ex:
+
+---
+Subject: validate-user,user-email=user@host
+---
+
+You may add body arguments separated by <\r\n>, ex:
+
+---
+Subject: validate-user
+
+user-mail=user@host
+another-argument=something else
+---

lib/distributor/templates/list_users.txt.erb

@@ -0,0 +1,5 @@
+List of the email subscribed to <%= list.name %> <<%= list.email %>>:
+
+<% list.emails.each do |email| %>
+  - <%= email.name %> <<%= email.email %>> (permissions=<%= email.permissions %>, created <%= email.created_at %>)
+<% end %>

lib/distributor/templates/set_permissions.success.txt.erb

@@ -0,0 +1,5 @@
+You are subscribed to <%= list.name %> <<%= list.email %>>
+
+The subscription has been updated by a modo or administration.
+
+Your updated permissions are <%= user.permissions_words.join(", ") %>

lib/distributor/templates/subscribe.forbidden.txt.erb

@@ -0,0 +1 @@
+Sorry subscribe failed. You may not have permissions to do that or already registered.

lib/distributor/templates/subscribe.success.txt.erb

@@ -0,0 +1,3 @@
+You have subscribed to <%= list.name %> <<%= list.email %>>
+
+Your current permissions are <%= register.permissions_words.join(', ') %>

lib/distributor/templates/subscribe.wait_modo.txt.erb

@@ -0,0 +1,10 @@
+<%= register.email %> registered to <%= list.email %>
+
+However the user has permissions to NONE for now and cannot read or write on the ML.
+
+You can set the permissions of the users to:
+
+- Normal read&write: <%= list.set_permissions_email(:reader, :writer, user_email: register.email) %>
+- Simple reader: <%= list.set_permissions_email(:reader, user_email: register.email) %>
+- Modo that can change members permissions (except op): <%= list.set_permissions_email(:reader, :writer, :modo, user_email: register.email) %>
+<% if modo.op? %> - Operator that can change members permissions (all): <%= list.set_permissions_email(:reader, :writer, :operator, :modo, user_email: register.email) %><% end %>

lib/distributor/templates/subscribe.wait_user.txt.erb

@@ -0,0 +1,3 @@
+You have subscribed to <%= list.name %> <<%= list.email %>>
+
+A modo will confirm your subscription.

lib/distributor/templates/unsubscribe.success.txt.erb

@@ -0,0 +1,3 @@
+You unsubscribed from <%= list.name %>.
+
+We won't mail you anymore and you are removed permanently from our database.

lib/distributor/wait_validation.txt.erb

@@ -0,0 +1,3 @@
+You have subscribed to <%= list.name %> <<%= list.email %>>
+
+A modo will confirm your subscription.

lib/distributor/wait_validation_modo.txt.erb

@@ -0,0 +1,3 @@
+<%= register.email %> registered to <%= list.email %>
+
+You can confirm with <%= list.validate_user_email(to.from) %>

lib/logger.rb

@@ -0,0 +1,31 @@
+require "logger"
+
+class MultiIO
+  def initialize(*targets)
+    @targets = targets
+  end
+
+  def <<(io)
+    @targets << io unless io == self
+    self
+  end
+
+  def write(*args)
+    @targets.each { _1.write(*args) }
+    self
+  end
+
+  def close
+    @targets.each(&:close)
+  end
+end
+
+log_output = MultiIO.new(STDOUT)
+if ENV["LOG_FILE"]
+  log_file = File.open(ENV["LOG_FILE"], "a")
+  log_output << log_file
+end
+
+$logger = Logger.new(log_output)
+$logger.level = Logger::INFO
+$logger.level = Logger::DEBUG if $debug

lib/models.rb

@@ -0,0 +1,3 @@
+require "sequel"
+require_relative "models/mailinglist"
+require_relative "models/email"

lib/models/email.rb

@@ -0,0 +1,107 @@
+class Email < Sequel::Model($db)
+  many_to_one :mailinglist
+
+  def before_create
+    self.created_at ||= Time.now
+    super
+  end
+
+  def before_save
+    self.updated_at ||= Time.now
+    super
+  end
+
+  module Permissions
+    NONE  = 0    # 0
+    READ  = 2**0 # 1
+    WRITE = 2**1 # 2
+    OP    = 2**2 # 4
+    MODO  = 2**3 # 8
+    ALL   = READ | WRITE | OP | MODO
+    WORDS = {
+      READ  => "reader",
+      WRITE => "writer",
+      OP    => "operator",
+      MODO  => "modo",
+    }.freeze
+    FROM_SYMBOLS = {
+      reader:   READ,
+      writer:   WRITE,
+      operator: OP,
+      modo:     MODO,
+      none:     NONE,
+    }
+    def self.from_symbols(*syms)
+      syms.map { FROM_SYMBOLS[_1] }.sum
+    end
+
+    def self.op?(perm)
+      perm & OP == OP
+    end
+  end
+
+  def permissions?(ask)
+    (permissions & ask) == ask
+  end
+
+  def reader?
+    permissions?(Permissions::READ)
+  end
+
+  def writer?
+    permissions?(Permissions::WRITE)
+  end
+
+  def op?
+    permissions?(Permissions::OP)
+  end
+
+  def modo?
+    permissions?(Permissions::MODO)
+  end
+
+  def permissions_words
+    words = Permissions::WORDS.filter { permissions?(_1) }.values
+    words << "none" if words.empty?
+    words
+  end
+
+  # Adds a new email to a ML, and respect the ML strategy.
+  # Registers with full permissions if first subscribe.
+  #
+  # @param name
+  # @param email
+  # @param mailinglist required if mailinglist.nil?
+  # @param mailinglist_id required if mailinglist.nil?
+  def self.register!(name:, email:, mailinglist: nil, mailinglist_id: nil)
+    mailinglist = Mailinglist.first(id: mailinglist_id) if mailinglist.nil?
+    if mailinglist.nil?
+      $logger.error "No mailing list found with id=#{mailinglist_id}"
+      raise "No mailinglist #{mailinglist_id}"
+    end
+
+    permissions =
+      if mailinglist.emails.count == 0
+        Permissions::ALL
+      elsif mailinglist.registration == "free"
+        Permissions::READ | Permissions::WRITE
+      elsif mailinglist.registration == "validated"
+        Permissions::NONE
+      elsif mailinglist.registration == "closed"
+        $logger.warn "Forbidden register for #{mailinglist.id}"
+        return nil # not valid
+      end
+
+    $logger.debug { "Existing users: #{mailinglist.emails.map(&:email).join(', ')}" }
+    new(
+      name: name,
+      email: email,
+      mailinglist: mailinglist,
+      permissions: permissions,
+    ).save
+  end
+
+  def self.unregister!(mailinglist:, email:)
+    Email.where(mailinglist: mailinglist, email: email).delete
+  end
+end

lib/models/mailinglist.rb

@@ -0,0 +1,129 @@
+require "uuid"
+
+class Mailinglist < Sequel::Model($db)
+  SUFFIX_SEPARATOR = ENV["MAILINGLIST_SUFFIX_SEPARATOR"] || "+"
+  BASE_USER = ENV["MAILINGLIST_BASE_USER"] || "mailinglist"
+  HOST = ENV["MAILINGLIST_HOST"]
+
+  STRATEGIES = {
+    registration: %w[autoregister free validated closed],
+    moderation: %w[freewrite restrictedwrite],
+  }.freeze
+
+  one_to_many :emails
+
+  def before_create
+    self.created_at ||= Time.now
+    super
+  end
+
+  def before_save
+    self.updated_at ||= Time.now
+    super
+  end
+
+  def self.build(name:, aliasing: nil, suffix: nil, strategy: "closed|freewrite")
+    email = BASE_USER.dup
+    email << SUFFIX_SEPARATOR << suffix if suffix
+    aliasing = UUID.generate if !suffix && !aliasing
+    email << aliasing if aliasing
+    email << "@#{HOST}"
+
+    new(
+      name: name,
+      email: email,
+      count_handled: 0,
+      count_distributed: 0,
+      last_email: nil,
+      strategy: strategy,
+    )
+  end
+
+  def self.search_mail(mail)
+    find(email: mail.to)
+  end
+
+  def signature
+    "\r\n\r\n---\r\nYou can unsubscribe to the mailinglist #{name} via: <mailto:#{unsubscribe_email}"
+  end
+
+  def registration
+    strategy.split("|").filter { STRATEGIES[:registration].include?(_1) }
+  end
+
+  def registration?(find)
+    registration.include?(find)
+  end
+
+  def moderation
+    strategy.split("|").filter { STRATEGIES[:moderation].include?(_1) }
+  end
+
+  def moderation?(find)
+    moderation.include?(find)
+  end
+
+  def enabled_readers
+    emails.filter { _1.permissions & Email::Permissions::READ != 0 }
+  end
+
+  def enabled_writers
+    emails.filter { _1.permissions & Email::Permissions::WRITE != 0 }
+  end
+
+  def enabled_ops
+    emails.filter { _1.permissions & Email::Permissions::OP != 0 }
+  end
+
+  def enabled_modos
+    emails.filter { _1.permissions & Email::Permissions::MODO != 0 }
+  end
+
+  def actions_emails
+    ACTIONS_EMAILS.to_h { [_1, send("#{_1}_email")] }
+  end
+
+  ACTIONS_EMAILS = %i[
+    help subscribe unsubscribe owner
+    set_permissions list_users
+  ].freeze
+  def help_email
+    "#{email}?subject=help"
+  end
+
+  def subscribe_email
+    "#{email}?subject=subscribe"
+  end
+
+  def unsubscribe_email
+    "#{email}?subject=unsubscribe"
+  end
+
+  def post_email
+    email
+  end
+
+  def owner_email
+    enabled_ops.first.email
+  end
+
+  def validate_distribute_email(uid)
+    "#{email}?subject=validate,uid=#{uid}"
+  end
+
+  def refuse_distribute_email(uid)
+    "#{email}?subject=refuse,uid=#{uid}"
+  end
+
+  def set_permissions_email(*permissions_symbols, user_email: nil, permissions: nil)
+    permissions = Email::Permissions.from_symbols(*permissions_symbols) if permissions.nil?
+    user_part = user_email ? ",user-email=#{user_email}" : ""
+    permissions_part = ",permissions=#{permissions}"
+    "#{email}?subject=set-permissions#{user_part}#{permissions_part}"
+  end
+
+  def list_users_email
+    "#{email}?subject=list-users"
+  end
+
+end

lib/protocols.rb

@@ -0,0 +1,20 @@
+# Protocols is a layer that help to interact with the ruby implementation of
+# IMAP, SMTP, and the mail standard.
+#
+# @example
+#     @smtp_client = Protocols::Smtp.new
+#     @imap_client = Protocols::Imap.new
+#     mail = Protocols::Mail.new(imap_mail: imap_mail)
+module Protocols
+  FULL     = "RFC822".freeze
+  HEADERS  = "RFC822.HEADER".freeze
+  BODYTEXT = "BODY[TEXT]".freeze
+  ENVELOPE = "ENVELOPE".freeze
+  UID      = "UID".freeze
+  SEQ      = "BODY[HEADER.FIELDS (X-SEQUENCE)]".freeze
+  MSG_ID   = "BODY[HEADER.FIELDS (MESSAGE-ID)]".freeze
+end
+
+require_relative "protocols/mail"
+require_relative "protocols/smtp"
+require_relative "protocols/imap"

lib/protocols/imap.rb

@@ -0,0 +1,183 @@
+require "net/imap"
+
+# Protocls::Imap allows to fetch emails.
+#
+# https://www.rfc-editor.org/rfc/rfc3501
+#
+# TODO: strengthen the network management to avoid connection loss.
+class Protocols::Imap
+  MODERATION_INBOX = "INBOX/moderation".freeze
+  REFUSED_INBOX = "INBOX/refused".freeze
+  BASE_INBOX = "INBOX".freeze
+  ALL_INBOXES = [MODERATION_INBOX, REFUSED_INBOX, BASE_INBOX].freeze
+
+  def initialize
+    reset!
+  end
+  attr_reader :imap if $debug
+
+  def reset!
+    @imap = Net::IMAP.new(ENV["IMAP_HOST"], port: ENV["IMAP_PORT"], ssl: true)
+    @imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
+    ALL_INBOXES.each do
+      $logger.info "Try to create inbox #{_1}"
+      @imap.create(_1)
+      $logger.info "Inbox #{_1} created"
+    rescue StandardError => e
+      $logger.error(e.message)
+    end # TODO: properly do that
+  end
+
+  # Fetch the next incomming email as a Protocols::Mail.
+  #
+  # @return [Protocols::Mail]
+  def fetch_next_unseen(inbox:, max_tries: 2)
+    return $logger.error("fetch_next_unseen reached max_tries_limit") if max_tries == 0
+
+    begin
+      uid = search_unseen(inbox:).first
+    rescue IOError => e
+      $logger.warn e.message
+      reset!
+      return fetch_next_unseen(inbox:, max_tries: max_tries - 1)
+    end
+
+    return nil if uid.nil?
+    fetch(uid:, inbox:)
+  end
+
+  # Fetch all UID that are not marked as SEEN yet in the inbox
+  #
+  # @param inbox [String]
+  # @return [Array(Integer)]
+  def search_unseen(inbox:)
+    goto_inbox(inbox)
+    @imap.uid_search(%w[NOT SEEN])
+  end
+
+  # Fetch the first UID of message matching with the given message_id.
+  # Will return nil if none are matching.
+  # It is possible for several messages to have the same message_id.
+  # In this case it is impossible to say which one will be returned.
+  # In practice, it is likely that the older message will be always returned.
+  #
+  # @param inbox [String]
+  # @return [Integer?]
+  def search_message_id(message_id, inbox:)
+    goto_inbox(inbox)
+    @imap.uid_search(["HEADER", "Message-ID", message_id]).first
+  end
+
+  # Open an given inbox. Do not multiply requests to the IMAP server.
+  #
+  # @param inbox [String] the name of the inbox repository
+  # @param readonly [TrueClass, FalseClass] if true, next operations won't modify the box (with move, store...)
+  #
+  # @example
+  #
+  #   goto_inbox Protocols::Imap::BASE_INBOX
+  #   goto_inbox Protocols::Imap::MODERATION_INBOX, readonly: true
+  def goto_inbox(inbox, readonly: false)
+    return if @current_inbox == inbox
+
+    $logger.debug "goto_inbox #{inbox}"
+    select_method = readonly ? :examine : :select
+    @imap.send(select_method, inbox)
+    @imap.check
+    @current_inbox = inbox
+  end
+
+  # Fetch the first message by UID or ID if it exists.
+  #
+  # @param id [String?] optional, specify either id or uid
+  # @param uid [String?] optional, specify either id or uid
+  #
+  # @return [Protocols::Mail?]
+  def fetch_first(inbox:, uid: nil, id: nil)
+    goto_inbox(inbox)
+    fetch_method = id ? :fetch : :uid_fetch
+    imap_mail = @imap.send(
+      fetch_method,
+      id || uid,
+      [Protocols::ENVELOPE, Protocols::UID, Protocols::HEADERS, Protocols::BODYTEXT, Protocols::MSG_ID],
+    )&.first
+    return nil if imap_mail.nil?
+
+    $logger.debug imap_mail.attr["RFC822.HEADER"]
+    Protocols::Mail.new(imap_mail:)
+  end
+
+  # Fetch one incoming mail as a Protocols::Mail.
+  # {see #fetch_first}
+  #
+  # @param id  [String?]
+  # @param uid [String?]
+  #
+  # @example
+  #
+  #   fetch(uid: 1, inbox: Protocols::Imap::BASE_INBOX)
+  #
+  # @return [Protocols::Mail]
+  def fetch(inbox:, id: nil, uid: nil)
+    raise "Need id OR uid to be set" if !id && !uid
+
+    uid = search_unseen(inbox:).first
+    mail = fetch_first(uid:, inbox:)
+
+    $logger.info "READ #{id || uid} #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+
+    mail
+  end
+
+  # Move an email to the another inbox.
+  #
+  # @param inbox [String?]         inbox to take the email from. Do nothing if nil.
+  # @param mail  [Protocols::Mail] mail to move
+  # @param to    [String]          inbox repository where the mail is placed
+  def move(mail:, to:, inbox:)
+    goto_inbox inbox
+    $logger.info "MOVED #{mail.uid} from #{inbox} to #{to}"
+    @imap.uid_move mail.uid, to
+  end
+
+  # Move an email to the moderated inbox.
+  #
+  # @param mail [Protocols::Mail]
+  def moderate(mail, inbox: BASE_INBOX)
+    move mail:, to: MODERATION_INBOX, inbox:
+  end
+
+  # # Remove an email from the moderated inbox.
+  # #
+  # # @param mail [Protocols::Mail]
+  # def unmoderate(mail)
+  #   @imap.uid_move mail.uid, BASE_INBOX
+  # end
+
+  # Add the SEEN flag to a given email.
+  # This email will not be reprocessed again.
+  #
+  # @param imap_mail [Protocols::Mail]
+  def seen!(imap_mail, inbox:)
+    goto_inbox inbox
+    $logger.info "MARK #{imap_mail.attr['UID']} as SEEN"
+    @imap.uid_store imap_mail.attr["UID"], "+FLAGS", [Net::IMAP::SEEN]
+  end
+
+  # Mark all existing messages as SEEN. See {#seen!}
+  def see_all_messages!
+    $logger.info "MARK ALL NOT SEEN as SEEN"
+    goto_inbox BASE_INBOX
+    see_all!
+    goto_inbox MODERATION_INBOX
+    see_all!
+  end
+
+  # :nodoc:
+  private def see_all!
+    @imap.uid_search(%w[NOT SEEN]).each do
+      @imap.uid_store _1, "+FLAGS", [Net::IMAP::SEEN]
+      $logger.debug "MARK #{_1} as SEEN"
+    end
+  end
+end

lib/protocols/mail.rb

@@ -0,0 +1,270 @@
+require "uuid"
+
+# Protocls::Mail provides an interface to read the content,
+# reply and redistribute the mail.
+# It handles multiple headers (having several times the same header key)
+# but you must know that gmail seem to do not allow it.
+class Protocols::Mail
+  attr_accessor :from_name, :from, :to_name, :to, :subject, :body, :uid, :message_id
+
+  DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z".freeze
+  HEADERS_KEEP = %w[
+    Content-Type Content-Transfer-Encoding MIME-Version
+  ].freeze
+  HEADERS_KEEP_WITH_ORIGINAL = HEADERS_KEEP + %w[Message-ID User-Agent From To Subject Subject Date In-Reply-To
+                                                 References]
+  USER_AGENT = "Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}) Mailinglist.rb/1.0".freeze
+  SENDER = ENV["SENDER"]
+  FROM = ENV["FROM"]
+
+  # @params imap_mail is any element outputed in the returned array of
+  #   Net::IMAP#fetch, and must contains some attributes:
+  #   - Protocols::ENVELOPE
+  #   - Protocols::HEADERS
+  #   - Protocols::BODYTEXT
+  #   - Protocols::UID
+  def initialize(imap_mail: nil)
+    if imap_mail
+      @imap_mail  = imap_mail
+      envelope    = imap_mail.attr[Protocols::ENVELOPE]
+      body_head   = imap_mail.attr[Protocols::HEADERS]
+      body_text   = imap_mail.attr[Protocols::BODYTEXT]
+      @from_name  = envelope.from.first.name
+      @to_name    = envelope.to.first.name
+      @from       = "#{envelope.from.first.mailbox}@#{envelope.from.first.host}"
+      @to         = "#{envelope.to.first.mailbox}@#{envelope.to.first.host}"
+      @subject    = envelope.subject
+      @body       = body_text
+      @headers    = Protocols::Mail.parse_rfc822_headers(body_head)
+      @headers.filter! { HEADERS_KEEP_WITH_ORIGINAL.include?(_1[0]) }
+      @seq        = header("X-Sequence").to_i
+      @message_id = header("Message-id")
+      @uid        = imap_mail.attr[Protocols::UID]
+    end
+  end
+  attr_reader :imap_mail if $debug
+
+  # Copy an existing mail.
+  # It will NOT work if the email do not contain a imap_mail.
+  def clone
+    Protocols::Mail.new(imap_mail: @imap_mail)
+  end
+
+  # Interface with Protocols::Mail
+  def seen!(imap_client:, inbox:)
+    if @imap_mail
+      imap_client.seen!(@imap_mail, inbox:)
+    else
+      $logger.debug("Cannot mark #{uid} as seen because no @imap_mail")
+      false
+    end
+  end
+
+  # This method is a kind of hash.
+  def cache_id
+    "#{@message_id} #{@from} #{@to}"
+  end
+
+  # Transform an received email into a mailing list email.
+  # Tested with evolution and Kmail.
+  # Thunderbird bug at the moment which make it weird but it works too.
+  #
+  # @param list [Mailinglist]
+  # @param dest [Email]
+  def to_redistribute(list:, dest:)
+    new = clone
+    sender =
+      if SENDER == "true"
+        @from
+      elsif SENDER == "list"
+        list.email
+      else
+        SENDER
+      end
+
+    from_email =
+      if FROM == "true"
+        @from
+      elsif FROM == "list"
+        list.email
+      else
+        FROM
+      end
+
+    from = "\"#{@from_name}\" (via #{list.name}) <#{from_email}>"
+
+    new.replace_headers!(
+      ["User-Agent",       USER_AGENT],
+      # require people to do not respond privatly
+      ["Reply-to",         "#{list.name} <#{list.email}>"],
+      ["Errors-To",        list.email],
+      ["To",               dest.email],
+      ["From",             from],
+      ["Sender",           sender ],
+      ["Date",             Time.now.strftime(DATE_FORMAT)],
+      ["List-Id",          "<#{list.email}>"],
+      ["List-Post",        "<mailto:#{list.email}>"],
+      ["List-Help",        "<mailto:#{list.help_email}>"],
+      ["List-Subscribe",   "<mailto:#{list.subscribe_email}>"],
+      ["List-Unsubscribe", "<mailto:#{list.unsubscribe_email}>"],
+      ["List-Post",        "<mailto:#{list.post_email}>"],
+      ["List-Owner",       "<mailto:#{list.owner_email}>"],
+      ["In-Reply-To",      @message_id],
+      %w[Precedence list],
+      # ["Precedence",       "bulk"],
+      ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
+      ["X-Sequence",         (@seq + 1).to_s],
+      ["X-Loop",           list.email],
+      ["Return-Path",      "<#{list.email}>"],
+    )
+
+    new.to   = dest.email
+    new.from = list.email
+    new.body = "#{body}#{list.signature}"
+
+    new
+  end
+
+  # Create a email from scratch.
+  #
+  # @param list [Mailinglist]
+  # @param to   [String]        email of the target
+  def self.build(subject:, list:, to:, body:, replace_headers: [])
+    new = new()
+    new.replace_headers!(
+      ["User-Agent",       USER_AGENT],
+      ["Reply-to",         "#{list.name} <#{list.email}>"],
+      ["Errors-To",        list.email],
+      ["To",               to],
+      ["Subject",          subject],
+      ["From",             "\"#{list.name}\" <#{list.email}>"],
+      ["Sender",           list.email],
+      ["Date",             Time.now.strftime(DATE_FORMAT)],
+      ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
+    )
+    new.replace_headers!(*replace_headers) if replace_headers
+
+    new.subject = subject
+    new.to   = to
+    new.from = list.email
+    new.body = "#{body}#{list.signature}"
+    new
+  end
+
+  # Create a email from scratch.
+  # Create a response email for an existing one.
+  #
+  # @param list [Mailinglist]
+  # @param mail [Protocols::Mail]
+  def to_response(list:, mail:, body:)
+    new = clone
+    new.replace_headers!(
+      ["User-Agent",       USER_AGENT],
+      # require people to do not respond privatly
+      ["Reply-to",         "#{list.name} <#{list.email}>"],
+      ["Errors-To",        list.email],
+      ["To",               list.email],
+      ["From",             "\"#{list.name}\" <#{list.email}>"],
+      ["Sender",           list.email],
+      ["Date",             Time.now.strftime(DATE_FORMAT)],
+      ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
+    )
+
+    new.to   = mail.from
+    new.from = list.email
+    new.body = "#{body}#{list.signature}"
+    new
+  end
+
+  # :nodoc:
+  def self.parse_rfc822_headers(rfc822)
+    headers = []
+    rfc822.split("\r\n").each do |line|
+      if line[0] == "\t"
+        headers.last[1] << "\r\n" << line
+      else
+        k, v = line.split(":", 2)
+        headers << [k, v.to_s.strip]
+      end
+    end
+    headers
+  end
+
+  # Get a header by key. Case insensitive. Only return the first occurence.
+  #
+  # @return [String?]
+  def header(k)
+    tuple = headers.find { _1[0].downcase == k.downcase }
+    tuple && tuple[1]
+  end
+
+  # Getter the all the parsed headers.
+  #
+  # @return [Array(Tuple(String, String))]
+  def headers
+    @headers ||= []
+  end
+
+  # Generate a list of headers
+  def kept_headers
+    headers.filter { HEADERS_KEEP.include?(_1[0]) }
+  end
+
+  # Add new headers. Do not overwrite existing ones.
+  #
+  # @params new_headers is a [ [key, value], ...] array
+  def add_headers!(*new_headers)
+    new_headers.each { headers << _1 }
+  end
+
+  # Remove existing headers.
+  #
+  # @param headers_keys is a [word, word, ...] array
+  def delete_headers!(*headers_keys)
+    headers_keys.each { delete_header! _1 }
+  end
+
+  # :nodoc:
+  private def delete_header!(header_key)
+    headers.delete_if { _1[0].downcase == header_key.downcase }
+  end
+
+  # Remove conflicting headers and replace them with a new set.
+  # If the header do not exist yet it is simply added.
+  #
+  # @params update_headers is a [ [key, value], ...] array
+  def replace_headers!(*update_headers)
+    delete_headers!(*update_headers.map(&:first))
+    add_headers!(*update_headers)
+  end
+
+  # TODO
+  # List-Id: <mailto:>
+  # List-Help: <mailto:>
+  # List-Subscribe: <mailto:>
+  # List-Unsubscribe: <mailto:>
+  # List-Post: <mailto:>
+  # List-Owner: <mailto:>
+  # List-Archive: <https://...>
+  # Archived-At: <https://...>
+  # List-Post: NO (posting not allowed on this list)
+
+  # :nodoc:
+  private def smtp_headers
+    headers.map { _1.join(": ") }.join("\r\n")
+  end
+
+  # Generate a Truple(String, String, String) that can be sent
+  # directly to the SMTP server using Net::SMTP or Protocols::SMTP.
+  #
+  # Notes:
+  # - https://www.ietf.org/rfc/rfc2369.txt
+  # - https://www.rfc-editor.org/rfc/rfc5983
+  def to_smtp
+    [
+      "#{smtp_headers}\r\n\r\n#{@body}",
+      @from,
+      @to,
+    ]
+  end
+end

lib/protocols/smtp.rb

@@ -0,0 +1,69 @@
+require "net/smtp"
+
+# High level Interface to distribute Protocols::Mail objects.
+# It is robust against network loss.
+class Protocols::Smtp
+  def initialize
+    reset_smtp_client!
+  end
+
+  class DistributedCache < Array
+    def initialize(max_size: 4096)
+      @max_size = max_size
+      super()
+    end
+
+    def <<(message_id)
+      prepend(message_id) if !message_id.nil?
+      slice! @max_size
+    end
+  end
+
+  def cache
+    (@cache ||= DistributedCache.new)
+  end
+
+  # @param mail [Protocols::Mail]
+  def distribute(mail)
+    if cache.include?(mail.cache_id)
+      $logger.warn "Already distributed #{mail.message_id}"
+      return
+    end
+
+    $logger.info "SEND #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    smtp_raw = mail.to_smtp
+    $logger.debug smtp_raw.join("=====")
+    send_message_safe(*smtp_raw)
+    cache << mail.cache_id
+  rescue StandardError => e
+    $logger.warn "FAILED #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    $logger.debug e
+  end
+
+  # :nodoc:
+  private def reset_smtp_client!
+    @smtp = Net::SMTP.new(
+      ENV["SMTP_HOST"], ENV["SMTP_PORT"], tls: ENV["SMTP_TLS"] == "true",
+    ).start(
+      user: ENV["SMTP_USER"],
+      secret: ENV["SMTP_PASSWORD"],
+      authtype: :login,
+    )
+  end
+
+  # :nodoc:
+  private def send_message_safe(*raw, max_tries: 3)
+    return $logger.error("send_message_safe reached max_tries_limit") if max_tries == 0
+
+    begin
+      @smtp.send_message(*raw)
+    rescue EOFError, Net::SMTPServerBusy => e
+      $logger.warn e.message
+      reset_smtp_client!
+      send_message_safe(*raw, max_tries: max_tries - 1)
+    rescue => e
+      $logger.error e.full_message
+      reset_smtp_client!
+    end
+  end
+end

lib/web.rb

@@ -0,0 +1,2 @@
+require "sinatra"
+require_relative "web/mailinglists"

lib/web/mailinglists.rb

@@ -0,0 +1,9 @@
+get "/" do
+  mailinglists = Mailinglist.where(strategy: %w[free validated]).all
+  slim :index, locals: {mailinglists:}
+end
+
+get "/mailinglist/:id" do
+  mailinglist = Mailinglist.first(id: params[:id], strategy: %w[free validated])
+  slim :mailinglist, locals: {mailinglist:}
+end

lib/web/views/index.slim

@@ -0,0 +1,10 @@
+h1
+  | Index
+p
+  ul
+  - for mailinglist in mailinglists
+    li
+      a href="/mailinglist/#{mailinglist.id}"
+        strong
+          | #{mailinglist.name}
+      | : #{mailinglist.email}

lib/web/views/layout.slim

@@ -0,0 +1,8 @@
+doctype html
+html
+  head
+    title
+      | Mailing List
+
+  body
+    == yield

lib/web/views/mailinglist.slim

@@ -0,0 +1,9 @@
+h1
+  | Mailinglist #{mailinglist.name}
+p
+  ul
+  - for email in mailinglist.emails
+    li
+      strong
+        | #{email.name}
+      | : #{email.email}

server.rb

@@ -1,96 +0,0 @@
-require "net/smtp"
-require "net/imap"
-require "dotenv"
-require "uuid"
-require "pry"
-
-Dotenv.load!
-
-class MailingList
-  class Smtp
-    def initialize
-      @smtp = Net::SMTP.new(
-        ENV["SMTP_HOST"], ENV["SMTP_PORT"], tls: ENV["SMTP_TLS"] == "true",
-      ).start(
-        user: ENV["SMTP_USER"],
-        secret: ENV["SMTP_PASSWORD"],
-        authtype: :login,
-      )
-    end
-
-    def distribute(*mail)
-      @smtp.send_message(*mail)
-    end
-  end
-
-  class Imap
-    def initialize
-      @imap = Net::IMAP.new(ENV["IMAP_HOST"], ENV["IMAP_PORT"], ssl: true)
-      @imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
-      @imap.select("INBOX")
-    end
-
-    BODY = "RFC822".freeze
-    ENVELOPE = "ENVELOPE".freeze
-    def fetch
-      @imap.check
-      # puts "DEBUG: check"
-      id = @imap.search(["NOT", "SEEN"]).first
-      return nil if id.nil?
-      # puts "DEBUG: id=#{id}"
-      message_raw = @imap.fetch(id, [ENVELOPE, BODY]).first
-      return nil if message_raw.nil?
-      message = message_raw.attr
-      # puts "DEBUG: message found"
-      from_name = message[ENVELOPE].from.first.name
-      to_name = message[ENVELOPE].to.first.name
-      from = "#{message[ENVELOPE].from.first.mailbox}@#{message[ENVELOPE].from.first.host}"
-      to = "#{message[ENVELOPE].to.first.mailbox}@#{message[ENVELOPE].to.first.host}"
-      subject = message[ENVELOPE].subject
-      body = message[BODY].split("\r\n\r\n", 2).last
-      # puts "DEBUG: generate mail"
-      mail = Mail.new(from_name:, from:, to_name:, to:, subject:, body:)
-
-      @imap.store id, "+FLAGS", [Net::IMAP::SEEN]
-      puts "#{from}\t -> #{to}:\t#{subject}"
-      # puts "DEBUG: set as seen"
-      mail
-    end
-  end
-
-  class Mail
-    attr_accessor :from_name, :from, :to_name, :to, :subject, :body
-
-    def initialize(from_name:, from:, to_name:, to:, subject:, body:)
-      @from_name = from_name
-      @from = from
-      @to_name = to_name
-      @to = to
-      @subject = subject
-      @body = body
-    end
-
-    def to_smtp_distribution(to:)
-      [
-"From: Mailinglist <#{@to}>
-To: <#{to}>
-Subject: #{@subject}
-Date: #{Time.now.to_s}
-Message-Id: <#{UUID.generate}@example.com>
-
-#{@body}",
-        @to,
-        to,
-      ]
-    end
-  end
-end
-
-smtp = MailingList::Smtp.new
-imap = MailingList::Imap.new
-
-while true
-  mail = imap.fetch
-  smtp.distribute(*mail.to_smtp_distribution(to: "arthur.poulet.test@sceptique.eu")) if mail
-  sleep 1
-end

test/protocols/test_mail.rb

@@ -0,0 +1,51 @@
+require "test_helper"
+require "protocols"
+
+module Protocols
+  class MailTest < Minitest::Test
+    def setup
+      @headers = <<~MAILEND
+        From: SenderH <sender@head.local>\r
+        To: DestH <dest@head.local>\r
+        Subject: -subject.head-\r
+        Date: #{DateTime.parse('2022-02-03T12:30:00Z').strftime(Protocols::Mail::DATE_FORMAT)}\r
+        Content-Type: plain/text\r
+        User-Agent: minitest\r
+      MAILEND
+      @imap_mail = OpenStruct.new(
+        attr: {
+          Protocols::ENVELOPE => OpenStruct.new(
+            from: [OpenStruct.new(mailbox: "sender", host: "local", name: "Sender")],
+            to: [OpenStruct.new(mailbox: "dest", host: "local", name: "Dest")],
+            subject: "-subject-",
+          ),
+          Protocols::HEADERS => @headers,
+          Protocols::BODYTEXT => "Content of the mail",
+        },
+      )
+      super
+    end
+
+    def test_parse_rfc822_headers
+      h1 = Mail.parse_rfc822_headers("Key: Value\r\n")
+      assert_equal 1, h1.size
+      h1.each { |tuple| assert_equal 2, tuple.size }
+      assert_equal "Key", h1.first.first
+      assert_equal "Value", h1.first.last
+
+      h2 = Mail.parse_rfc822_headers("Key: Value\r\n\tPartTwo\r\n")
+      assert_equal 1, h2.size
+      h2.each { |tuple| assert_equal 2, tuple.size }
+      assert_equal "Value\r\n\tPartTwo", h2.first.last
+    end
+
+    def test_init_with_imap_mail
+      mail = Mail.new(imap_mail: @imap_mail)
+      assert_equal "Sender", mail.from_name
+      assert_equal "sender@local", mail.from
+      assert_equal "Dest", mail.to_name
+      assert_equal "dest@local", mail.to
+      assert_equal "SenderH <sender@head.local>", mail.header("From")
+    end
+  end
+end

test/test_helper.rb

@@ -0,0 +1,3 @@
+require "pry"
+require "minitest"
+require "mocha/minitest"