Fix date format for better mail compliance

- Validation system is implemented as a simple mail when creating the
  email subscription; then it is simply a set-permissions
- Implement set-permissions
- Improve attribute parsing security and quality
- Improve emails clarity
- Improve permissions security
- Improve some logs for security
- Rename admin to operator to split the server admin from ML op
- Stop using inline functions as emacs ruby-mode do not support it yet

.gitignore

@@ -1 +1,4 @@
 .env
+*.db
+*.sqlite*
+README.html

Gemfile

@@ -2,11 +2,29 @@
 
 source "https://rubygems.org"
 
-# gem "rails"
-
+# Mails proto
 gem "net-smtp", "~> 0.3.3"
 gem "net-imap", "~> 0.3.1"
 
-gem "dotenv", "~> 2.8"
+# Mail template
+gem "erb", "~> 3.0"
 
+# Database
+gem "sqlite3", "~> 1.5"
+gem "sequel", "~> 5.62"
+gem "colorize", "~> 0.8.1"
+
+# Web
+gem "sinatra", "~> 3.0"
+gem "puma", "~> 6.0"
+gem "slim", "~> 4.1"
+
+# Others
+gem "dotenv", "~> 2.8"
 gem "uuid", "~> 2.3"
+gem "semver", "~> 1.0"
+
+# Debug and stuff
+group :develop do
+  gem "pry", "~> 0.14.1"
+end

Gemfile.lock

@@ -1,16 +1,47 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    cgi (0.3.3)
+    coderay (1.1.3)
+    colorize (0.8.1)
     dotenv (2.8.1)
+    erb (3.0.0)
+      cgi (>= 0.3.3)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
+    method_source (1.0.0)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
     net-imap (0.3.1)
       net-protocol
     net-protocol (0.1.3)
       timeout
     net-smtp (0.3.3)
       net-protocol
+    nio4r (2.5.8)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    puma (6.0.0)
+      nio4r (~> 2.0)
+    rack (2.2.4)
+    rack-protection (3.0.3)
+      rack
+    ruby2_keywords (0.0.5)
+    semver (1.0.1)
+    sequel (5.62.0)
+    sinatra (3.0.3)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.0.3)
+      tilt (~> 2.0)
+    slim (4.1.0)
+      temple (>= 0.7.6, < 0.9)
+      tilt (>= 2.0.6, < 2.1)
+    sqlite3 (1.5.3-x86_64-linux)
     systemu (2.6.5)
+    temple (0.8.2)
+    tilt (2.0.11)
     timeout (0.3.0)
     uuid (2.3.9)
       macaddr (~> 1.0)
@@ -19,9 +50,18 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  colorize (~> 0.8.1)
   dotenv (~> 2.8)
+  erb (~> 3.0)
   net-imap (~> 0.3.1)
   net-smtp (~> 0.3.3)
+  pry (~> 0.14.1)
+  puma (~> 6.0)
+  semver (~> 1.0)
+  sequel (~> 5.62)
+  sinatra (~> 3.0)
+  slim (~> 4.1)
+  sqlite3 (~> 1.5)
   uuid (~> 2.3)
 
 BUNDLED WITH

README.md

@@ -0,0 +1,73 @@
+# mailinglist.rb
+
+## Features
+
+- [x] Receive new message and send it
+- [x] Have a database of mailinglists with name and users
+- [ ] Web interface to access archives
+- [x] ACL to restrict interactions with the system
+- [x] Install documentation
+- [x] Systemd service
+- [ ] AUR package
+- [x] Configuration for everything
+- [x] Robust codebase for extensibility
+- [x] Robust network (handle network loss etc.)
+- [ ] Inbox cleanup
+- [ ] Moderation toolbox and email validation before distribution
+
+## Deploy from sources
+
+- Note there is a sample of systemd service you may use for your server in `/deploy/`
+
+### Download
+
+    git clone https://git.sceptique.eu/Sceptique/mailinglist.rb --depth 1
+    cd mailinglist.rb
+
+### Install dependencies
+
+System dependencies: `ruby 3.1.2`, `sqlite3`
+
+    bundle install
+
+### Configure
+
+Copy and fill all the env variables in .env
+
+    cp empty.env .env
+    edit .env
+
+### Setup the database
+
+    bin/db_migrate
+
+### Start
+
+You may test it with
+
+    bin/distributor
+
+### Dev & play localy
+
+After deploying it, there are some tools:
+
+- `bin/db_seed` to generate some data
+
+## Contributing
+
+### via Gitea
+
+1. Fork it (<https://git.sceptique.eu/Sceptique/mailinglistrb>)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+### via emails
+
+Checkout git-send-mail tutorial <https://git-send-email.io/>
+
+1. Clone it (<https://git.sceptique.eu/Sceptique/mailinglistrb>)
+2. Subscribe to the mailinglist to send your patch <mailto:list.mailinglistrb@sceptique.eu?subject=subscribe> (don't send your patch in this email)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Send your email to <mailto:list.mailinglistrb@sceptique.eu> after you are validated by modo

bin/db_console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "pry"
+
+# mailinglist1 = Mailinglist.build(name: "FreeML", suffix: "free", strategy: "free").save
+# email1 = Email.register!(name: "AP", email: "arthur.poulet.hunk@sceptique.eu", mailinglist: mailinglist)
+binding.pry

bin/db_mailinglist_create

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "optparse"
+require "uuid"
+
+options = {
+  name: UUID.generate,
+  strategy: "free",
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: db_mailinglist_create [options]"
+
+  opts.on("-s=STRATEGY", "--strategy=STRATEGY", "free/validated/closed") do |v|
+    options[:strategy] = v
+  end
+
+  opts.on("-n=NAME", "--name=NAME", "Define the name of the option") do |v|
+    options[:name] = v
+  end
+end.parse!
+
+options[:suffix] = options[:name].gsub(/[^a-zA-Z0-9]+/, '-')
+mailinglist = Mailinglist.build(name: options[:name], suffix: options[:suffix], strategy: options[:strategy]).save
+pp mailinglist

bin/db_migrate

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "semver"
+require "colorize"
+require "app"
+require "semver"
+
+def current_version
+  $db[:meta].first[:version] rescue 0
+end
+
+def migrate(version, message = nil, &block)
+  puts "Check migration #{version}".on_blue
+  if current_version < version
+    puts "Migrate #{version}".blue
+    begin
+      yield
+      $db[:meta].update(version: version)
+      puts "Successfuly set version #{version}".green
+      puts message.green if message
+    rescue => err
+      puts err.message.on_red
+      exit 1
+    end
+  else
+    puts "Already migrated #{version}".yellow
+  end
+end
+
+migrate 1, "Initialized database" do
+  # this table only contains one entry
+  $db.create_table :meta do
+    primary_key :id
+    Int :version
+    String :code_version
+    String :code_date
+  end
+  $db[:meta].insert(version: 1)
+end rescue puts "already initialized".yellow
+
+migrate 2, "Initialize mailinglists" do
+  $db.create_table :mailinglists do
+    primary_key :id
+    column :name,                  String, null: false # Name to display
+    column :email,                 String, null: false # Mailing list email
+    column :count_handled,         Integer  # amount of email distributed
+    column :count_distributed,     Integer  # amount of email distributed
+    column :last_email,            DateTime # date of last distribution
+    column :strategy,              String, null: false # "free/validated/closed/..."
+    column :updated_at,            DateTime
+    column :created_at,            DateTime
+
+    index :name, unique: true
+    index :email, unique: true
+  end
+end rescue puts "mailinglists already exists".yellow
+
+migrate 3, "Initialize emails" do
+  $db.create_table :emails do
+    primary_key :id
+    foreign_key :mailinglist_id, :mailinglists, null: false #, on_delete: :cascade
+    column :name,           String, null: false, fixed: true, size: 50
+    column :email,          String, null: false, fixed: true, size: 254
+    column :permissions,    Integer # bitfield. 1=read 2=write 4=admin
+    column :admin,          TrueClass
+    column :updated_at,     DateTime
+    column :created_at,     DateTime
+
+    index %i[email], unique: false # list personnal subscriptions
+    index %i[email mailinglist_id], unique: true
+    index %i[mailinglist_id], unique: false # allow to search entries by mailinglist
+    index %i[mailinglist_id permissions], unique: false # find readers, writers, etc.
+  end
+end rescue puts "emails already exists".yellow
+
+puts "End migration".on_blue
+code_version = `git tag`.split("\n").map{ |str| Semver.new(str) }.sort.last
+code_date = if code_version
+  `git show #{code_version} --pretty="format:%as"`.split("\n").first
+else
+  Time.now
+end
+code_version ||= "v0.0.0"
+$db[:meta].update(code_version: code_version.to_s)
+$db[:meta].update(code_date: code_date.to_s)
+puts "Set code version to #{code_version}".green
+puts "Set code date to #{code_date}".green

bin/db_seed

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+
+mailinglist1 = Mailinglist.build(name: "FreeML", suffix: "free", strategy: "free").save
+mailinglist2 = Mailinglist.build(name: "ValidatedML", suffix: "validated", strategy: "validated").save
+mailinglist3 = Mailinglist.build(name: "ClosedML", suffix: "closed", strategy: "closed").save
+pp mailinglist1, mailinglist2, mailinglist3
+
+# email1 = Email.register!(name: "AP", email: "arthur.poulet.hunk@sceptique.eu", mailinglist: mailinglist)
+# email2 = Email.register!(name: "AP2", email: "arthur.poulet.hunk2@sceptique.eu", mailinglist: mailinglist)
+# pp email1, email2

bin/distributor

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "distributor"
+
+Distributor.new.start(cpu_sleep: (ENV["CPU_SLEEP"] || 1).to_i)

bin/http

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.join(Dir.pwd, "lib")
+
+require "app"
+require "sinatra"
+set :views, File.expand_path(File.join(settings.root + "/../lib/web/views"))
+
+require "web"

deploy/mailinglistrb.service

@@ -0,0 +1,35 @@
+[Unit]
+Description=Mailinglist.rb
+Documentation=https://git.sceptique.eu/Sceptique/mailinglist.rb
+
+[Service]
+ExecStart=/opt/mailinglistrb/bin/distributor
+Restart=on-failure
+RestartSec=3
+User=mailinglistrb
+Group=mailinglistrb
+WorkingDirectory=/opt/mailinglistrb
+Environment=/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl
+
+SystemCallArchitectures=native
+CapabilityBoundingSet=
+NoNewPrivileges=true
+PrivateDevices=true
+RemoveIPC=true
+LockPersonality=true
+
+ProtectControlGroups=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectKernelLogs=true
+ProtectClock=true
+ProtectHostname=true
+ProtectProc=noaccess
+
+RestrictRealtime=true
+RestrictSUIDSGID=true
+RestrictNamespaces=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+
+[Install]
+WantedBy=default.target

empty.env

@@ -0,0 +1,18 @@
+SMTP_HOST=
+SMTP_PORT=465
+SMTP_TLS=true
+SMTP_USER=
+SMTP_PASSWORD=
+
+IMAP_HOST=
+IMAP_PORT=993
+IMAP_TLS=true
+IMAP_USER=
+IMAP_PASSWORD=
+
+SENDER_HOST=machin.fr
+DB_URL="sqlite://dev.db"
+
+PORT=10081
+DEBUG=false
+CPU_CYCLE=1

lib/app.rb

@@ -0,0 +1,23 @@
+require "dotenv"
+Dotenv.load!
+
+require "pry"
+$debug = ENV["DEBUG"] == "true"
+
+require "sequel"
+$db = Sequel.connect(ENV["DB_URL"])
+
+require_relative "logger"
+
+module Protocols
+  FULL     = "RFC822".freeze
+  HEADERS  = "RFC822.HEADER".freeze
+  BODYTEXT = "BODY[TEXT]".freeze
+  ENVELOPE = "ENVELOPE".freeze
+  UID      = "UID".freeze
+  SEQ      = "BODY[HEADER.FIELDS (X-SEQUENCE)]".freeze
+  MSG_ID   = "BODY[HEADER.FIELDS (MESSAGE-ID)]".freeze
+end
+
+require_relative "protocols"
+require_relative "models" rescue nil

lib/distributor.rb

@@ -0,0 +1,61 @@
+class Distributor
+
+  class Attributes < Hash
+    def self.parse(subject:, body:)
+      new = Attributes.new
+      subject_added_attribute = false
+      subject.to_s.split(",").each { new.add_attribute!(_1); subject_added_attribute = true }
+      body.to_s.split("\r\n").each { new.add_attribute!(_1) } if body.include?("=") && subject_added_attribute == false
+      new
+    end
+
+    def add_attribute!(key_eq_value)
+      k, v = key_eq_value.split("=", 2)
+      self[k.strip] = v.strip if k && v
+    end
+  end
+
+  def initialize
+    @smtp_client = Protocols::Smtp.new
+    @imap_client = Protocols::Imap.new
+    @imap_client.clean if ENV["HARD_RESET_NOT_SEEN_MESSAGE"] == "true"
+    @handlers = {
+      :default          => Actions::Distribute.new(distributor: self),
+      "subscribe"       => Actions::Subscribe.new(distributor: self),
+      "unsubscribe"     => Actions::Unsubscribe.new(distributor: self),
+      "help"            => Actions::Help.new(distributor: self),
+      "set-permissions" => Actions::SetPermissions.new(distributor: self),
+      "list-users"      => Actions::ListUsers.new(distributor: self),
+    }
+  end
+
+  def handle_one(mail)
+    $logger.info "incoming email from #{mail.from} | #{mail.subject}"
+    list = Mailinglist.search_mail(mail)
+    if list
+      # TODO: create list from mail ?
+      subject, subject_attributes = mail.subject.split(",", 2)
+      attributes = Attributes.parse(subject: subject_attributes, body: mail.body)
+      handler = @handlers[subject] || @handlers[:default]
+      $logger.info "#{handler.class}#handle on #{list.email} for #{mail.from}"
+      handler.handle(list: list, to: mail, attributes: attributes)
+    end
+    mail.seen!(imap_client: @imap_client)
+  end
+
+  def distribute(*ary, **opt)
+    @smtp_client.distribute(*ary, **opt)
+  end
+
+  def start(cpu_sleep: 1)
+    puts "fetching new mail to distribute every #{cpu_sleep} second..."
+    loop do
+      mail = @imap_client.fetch
+      handle_one(mail) if mail
+
+      sleep cpu_sleep
+    end
+  end
+
+  require_relative "distributor/action"
+end

lib/distributor/#subscribed.txt.erb#

@@ -0,0 +1 @@
+You have subscribed to <%= list.name %> <<%= list.email %>> 

lib/distributor/action.rb

@@ -0,0 +1,26 @@
+require "erb"
+
+class Distributor
+  module Actions
+
+    # Abstract
+    class Action
+      attr_reader :distributor
+
+      def initialize(distributor:)
+        @distributor = distributor
+      end
+
+      def handle(list:, to:, attributes:)
+        $logger.error "#{self.class} is not implemented yet"
+      end
+    end
+
+    def self.template(name)
+      ERB.new(File.read(File.join(__dir__, "templates", "#{name}.txt.erb")))
+    end
+  end
+end
+
+require_relative "actions/admin"
+require_relative "actions/user"

lib/distributor/actions/admin.rb

@@ -0,0 +1,46 @@
+class Distributor
+  module Actions
+
+    class SetPermissions < Action
+      SET_PERMISSIONS_TEMPLATE = Actions.template("set_permissions.success")
+
+      def handle(list:, to:, attributes:)
+        return if attributes["user-email"].nil? # drop missing param
+        return if attributes["permissions"].nil? # drop missing param
+
+        modo = Email.first(mailinglist: list, email: to.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{to.from}> failed to set-permissions <#{list.email}> modo"
+          return nil
+        end
+
+        user_email = attributes["user-email"]
+        user = Email.first(mailinglist: list, email: user_email)
+        if user.nil?
+          $logger.warn "SECU <#{to.from}> failed to set-permissions on non-existing email <#{user_email}>"
+          return nil
+        end
+
+        if user.op? && !modo.op?
+          $logger.warn "SECU <#{to.from}> failed to set-permissions on op email <#{user_email}>"
+          return nil
+        end
+
+        permissions = attributes["permissions"].to_i
+        if Email::Permissions.op?(permissions) && !modo.op?
+          $logger.warn "SECU <#{to.from}> failed to set op permissions on email <#{user_email}>"
+          return nil
+        end
+
+        user.permissions = permissions
+        user.save
+        body = SET_PERMISSIONS_TEMPLATE.result binding
+        @distributor.distribute(Protocols::Mail.build(subject: "ML #{list.name} subscription update", list: list, to: user.email, body: body))
+      end
+    end
+
+    class ListUsers < Action
+    end
+
+  end
+end

lib/distributor/actions/user.rb

@@ -0,0 +1,87 @@
+class Distributor
+  module Actions
+
+    class Subscribe < Action
+      FORBIDDEN_TEMPLATE = Actions.template("subscribe.forbidden")
+      SUCCESS_TEMPLATE = Actions.template("subscribe.success")
+      WAIT_USER_TEMPLATE = Actions.template("subscribe.wait_user")
+      WAIT_MODO_TEMPLATE = Actions.template("subscribe.wait_modo")
+
+      def handle(list:, to:, attributes:)
+        register =
+          begin
+            Email.register!(mailinglist: list, name: to.from_name, email: to.from).save
+          rescue => err
+            $logger.error err.message
+            nil
+          end
+        if register
+          if !register.reader?
+            handle_wait_validation(list: list, to: to, register: register)
+          else
+            handle_subscribed(list: list, to: to, register: register)
+          end
+        else
+          handle_403(list: list, to: to)
+        end
+      end
+
+      def handle_wait_validation(list:, to:, register:)
+        $logger.debug "Subscribe#handle_wait_validation on #{list.email} for #{to.from}"
+        body = WAIT_USER_TEMPLATE.result binding
+        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+
+        modo = list.enabled_modos.first
+        body = WAIT_MODO_TEMPLATE.result binding
+        @distributor.distribute(Protocols::Mail.build(subject: "ML #{list.name} requires validaton for #{to.from}", list: list, to: modo.email, body: body))
+      end
+
+      def handle_subscribed(list:, to:, register:)
+        $logger.debug "Subscribe#handle_subscribed on #{list.email} for #{to.from}"
+        $logger.debug register.inspect
+        body = SUCCESS_TEMPLATE.result binding
+        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+      end
+
+      def handle_403(list:, to:)
+        $logger.debug "Subscribe#handle_403 on #{list.email} for #{to.from}"
+        body = FORBIDDEN_TEMPLATE.result binding
+        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+      end
+    end
+
+    class Unsubscribe < Action
+      SUCCESS_TEMPLATE = Actions.template("unsubscribe.success")
+
+      def handle(list:, to:, attributes:)
+        Email.unregister!(mailinglist: list, email: to.from)
+        body = SUCCESS_TEMPLATE.result binding
+        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+      end
+    end
+
+    class Help < Action
+      HELP_TEMPLATE = Actions.template("help")
+      def handle(list:, to:, attributes:)
+        body = HELP_TEMPLATE.result binding
+        @distributor.distribute(to.to_response(list: list, to: to, body: body))
+      end
+    end
+
+    # This distribute the mail among the readers
+    class Distribute < Action
+      def handle(list:, to:, attributes:)
+        if !list
+          warn "invalid email writer for #{mail.from} on #{mail.to}"
+          return nil
+        end
+
+        list.enabled_readers.each do |reader|
+          to_distrib = to.to_redistribute(list: list, dest: reader)
+          @distributor.distribute(to_distrib)
+        end
+      end
+    end
+
+  end
+end

lib/distributor/templates/help.txt.erb

@@ -0,0 +1,25 @@
+# Mailing list: <%= list.name %> <<%= list.email %>>
+
+## Actions
+
+<% list.actions_emails.each do |action, action_email| %>
+- <%= action %> <mailto:<%= action_email %>><% end %>
+
+## User guide
+
+Most action are done via subject.
+Specify the action keyword in the subject or the body.
+You may add subjects arguments separated with <,>, ex:
+
+---
+Subject: validate-user,user-email=user@host
+---
+
+You may add body arguments separated by <\r\n>, ex:
+
+---
+Subject: validate-user
+
+user-mail=user@host
+another-argument=something else
+---

lib/distributor/templates/set_permissions.success.txt.erb

@@ -0,0 +1,5 @@
+You are subscribed to <%= list.name %> <<%= list.email %>>
+
+The subscription has been updated by a modo or administration.
+
+Your updated permissions are <%= user.permissions_words.join(", ") %>

lib/distributor/templates/subscribe.forbidden.txt.erb

@@ -0,0 +1 @@
+Sorry subscribe failed. You may not have permissions to do that or already registered.

lib/distributor/templates/subscribe.success.txt.erb

@@ -0,0 +1,3 @@
+You have subscribed to <%= list.name %> <<%= list.email %>>
+
+Your current permissions are <%= register.permissions.to_s(2).rjust(8, "0") %>

lib/distributor/templates/subscribe.wait_modo.txt.erb

@@ -0,0 +1,10 @@
+<%= register.email %> registered to <%= list.email %>
+
+However the user has permissions to NONE for now and cannot read or write on the ML.
+
+You can set the permissions of the users to:
+
+- Normal read&write: <%= list.set_permissions_email(:reader, :writer, user_email: register.email) %>
+- Simple reader: <%= list.set_permissions_email(:reader, user_email: register.email) %>
+- Modo that can change members permissions (except op): <%= list.set_permissions_email(:reader, :writer, :modo, user_email: register.email) %>
+<% if modo.op? %> - Operator that can change members permissions (all): <%= list.set_permissions_email(:reader, :writer, :operator, :modo, user_email: register.email) %><% end %>

lib/distributor/templates/subscribe.wait_user.txt.erb

@@ -0,0 +1,3 @@
+You have subscribed to <%= list.name %> <<%= list.email %>>
+
+A modo will confirm your subscription.

lib/distributor/templates/unsubscribe.success.txt.erb

@@ -0,0 +1,3 @@
+You unsubscribed from <%= list.name %>.
+
+We won't mail you anymore and you are removed permanently from our database.

lib/distributor/wait_validation.txt.erb

@@ -0,0 +1,3 @@
+You have subscribed to <%= list.name %> <<%= list.email %>>
+
+A modo will confirm your subscription.

lib/distributor/wait_validation_modo.txt.erb

@@ -0,0 +1,3 @@
+<%= register.email %> registered to <%= list.email %>
+
+You can confirm with <%= list.validate_user_email(to.from) %>

lib/logger.rb

@@ -0,0 +1,21 @@
+require "logger"
+
+$logger = Logger.new(STDOUT)
+$logger.level = Logger::INFO
+$logger.level = Logger::DEBUG if $debug
+
+def warn(*args, &block)
+  $logger.warn(*args, &block)
+end
+
+def puts(*args, &block)
+  $logger.info(*args, &block)
+end
+
+def error(*args, &block)
+  $logger.error(*args, &block)
+end
+
+def info(*args, &block)
+  $logger.info(*args, &block)
+end

lib/models.rb

@@ -0,0 +1,3 @@
+require "sequel"
+require_relative "models/mailinglist"
+require_relative "models/email"

lib/models/email.rb

@@ -0,0 +1,107 @@
+class Email < Sequel::Model($db)
+  many_to_one :mailinglist
+
+  def before_create
+    self.created_at ||= Time.now
+    super
+  end
+
+  def before_save
+    self.updated_at ||= Time.now
+    super
+  end
+
+  module Permissions
+    NONE  = 0    # 0
+    READ  = 2**0 # 1
+    WRITE = 2**1 # 2
+    OP    = 2**2 # 4
+    MODO  = 2**3 # 8
+    ALL   = READ | WRITE | OP | MODO
+    WORDS = {
+      READ  => "reader",
+      WRITE => "writer",
+      OP    => "operator",
+      MODO  => "modo",
+    }.freeze
+    FROM_SYMBOLS = {
+      reader:   READ,
+      writer:   WRITE,
+      operator: OP,
+      modo:     MODO,
+      none:     NONE,
+    }
+    def self.from_symbols(*syms)
+      syms.map { FROM_SYMBOLS[_1] }.sum rescue binding.pry
+    end
+
+    def self.op?(perm)
+      perm & OP == OP
+    end
+  end
+
+  def permissions?(ask)
+    (permissions & ask) == ask
+  end
+
+  def reader?
+    permissions?(Permissions::READ)
+  end
+
+  def writer?
+    permissions?(Permissions::WRITE)
+  end
+
+  def op?
+    permissions?(Permissions::OP)
+  end
+
+  def modo?
+    permissions?(Permissions::MODO)
+  end
+
+  def permissions_words
+    words = Permissions::WORDS.filter { permissions?(_1) }.values
+    words << "none" if words.empty?
+    words
+  end
+
+  # Adds a new email to a ML, and respect the ML strategy.
+  # Registers with full permissions if first subscribe.
+  #
+  # @param name
+  # @param email
+  # @param mailinglist required if mailinglist.nil?
+  # @param mailinglist_id required if mailinglist.nil?
+  def self.register!(name:, email:, mailinglist: nil, mailinglist_id: nil)
+    mailinglist = Mailinglist.first(id: mailinglist_id) if mailinglist.nil?
+    if mailinglist.nil?
+      $logger.error "No mailing list found with id=#{mailinglist_id}"
+      raise "No mailinglist #{mailinglist_id}"
+    end
+
+    permissions =
+      if mailinglist.emails.count == 0
+        Permissions::ALL
+      elsif mailinglist.strategy == "free"
+        Permissions::READ | Permissions::WRITE
+      elsif mailinglist.strategy == "validated"
+        Permissions::NONE
+      elsif mailinglist.strategy == "closed"
+        $logger.warn "Forbidden register for #{mailinglist.id}"
+        return nil # not valid
+      end
+
+    $logger.debug { "Existing users: #{mailinglist.emails.map(&:email).join(", ")}" }
+    new(
+      name: name,
+      email: email,
+      mailinglist: mailinglist,
+      permissions: permissions,
+    ).save
+  end
+
+  def self.unregister!(mailinglist:, email:)
+    Email.where(mailinglist: mailinglist, email: email).delete
+  end
+end

lib/models/mailinglist.rb

@@ -0,0 +1,107 @@
+require "uuid"
+
+class Mailinglist < Sequel::Model($db)
+  SUFFIX_SEPARATOR = ENV["MAILINGLIST_SUFFIX_SEPARATOR"] || "."
+  BASE_USER = ENV["MAILINGLIST_BASE_USER"] || "mailinglist"
+  HOST = ENV["MAILINGLIST_HOST"]
+
+  STRATEGIES = {
+    registration: %w[free validated closed],
+  }
+
+  one_to_many :emails
+
+  def before_create
+    self.created_at ||= Time.now
+    super
+  end
+
+  def before_save
+    self.updated_at ||= Time.now
+    super
+  end
+
+  def self.build(name:, aliasing: nil, suffix: nil, strategy: "closed")
+    email = "#{BASE_USER}"
+    email << SUFFIX_SEPARATOR << suffix if suffix
+    aliasing = UUID.generate if !suffix && !aliasing
+    email << aliasing if aliasing
+    email << "@#{HOST}"
+
+    new(
+      name: name,
+      email: email,
+      count_handled: 0,
+      count_distributed: 0,
+      last_email: nil,
+      strategy: strategy,
+    )
+  end
+
+  def self.search_mail(mail)
+    find(email: mail.to)
+  end
+
+  def signature
+    "\r\n\r\n---\r\nYou can unsubscribe to the mailinglist #{name} via: <mailto:#{unsubscribe_email}"
+  end
+
+  def enabled_readers
+    emails.filter{ _1.permissions & Email::Permissions::READ != 0 }
+  end
+
+  def enabled_writers
+    emails.filter{ _1.permissions & Email::Permissions::WRITE != 0 }
+  end
+
+  def enabled_admins
+    emails.filter{ _1.permissions & Email::Permissions::ADMIN != 0 }
+  end
+
+  def enabled_modos
+    emails.filter{ _1.permissions & Email::Permissions::MODO != 0 }
+  end
+
+  def actions_emails
+    ACTIONS_EMAILS.to_h { [_1, send("#{_1}_email")] }
+  end
+
+  ACTIONS_EMAILS = %i[
+    help subscribe unsubscribe owner
+    set_permissions list_users
+  ]
+  def help_email
+    "#{email}?subject=help"
+  end
+
+  def subscribe_email
+    "#{email}?subject=subscribe"
+  end
+
+  def unsubscribe_email
+    "#{email}?subject=unsubscribe"
+  end
+
+  def post_email
+    email
+  end
+
+  def owner_email
+    "#{email}?subject=owner" # TODO: first user?
+  end
+
+  def set_permissions_email(*permissions_symbols, user_email: nil, permissions: nil)
+    if permissions.nil?
+      permissions = Email::Permissions.from_symbols(*permissions_symbols)
+    end
+    user_part = user_email ? ",user-email=#{user_email}" : ""
+    permissions_part = ",permissions=#{permissions}"
+    "#{email}?subject=set-permissions#{user_part}#{permissions_part}"
+  end
+
+  def list_users_email
+    "#{email}?subject=list-users"
+  end
+
+
+end

lib/protocols.rb

@@ -0,0 +1,3 @@
+require_relative "protocols/mail"
+require_relative "protocols/smtp"
+require_relative "protocols/imap"

lib/protocols/imap.rb

@@ -0,0 +1,36 @@
+require "net/imap"
+
+class Protocols::Imap
+  def initialize
+    @imap = Net::IMAP.new(ENV["IMAP_HOST"], ENV["IMAP_PORT"], ssl: true)
+    @imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
+    @imap.select("INBOX")
+  end
+
+  def fetch
+    @imap.check
+    id = @imap.search(%w[NOT SEEN]).last
+    return nil if id.nil?
+
+    imap_mail = @imap.fetch(
+      id, [Protocols::ENVELOPE, Protocols::UID, Protocols::HEADERS, Protocols::BODYTEXT, Protocols::MSG_ID],
+    ).first
+    return nil if imap_mail.nil?
+
+    $logger.debug imap_mail.attr["RFC822.HEADER"]
+    mail = Protocols::Mail.new(imap_mail: imap_mail)
+    puts "READ #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+
+    mail
+  end
+
+  def clean
+    puts "MARK ALL NOT SEEN as SEEN"
+    @imap.search(["NOT", "SEEN"]).each { @imap.store _1, "+FLAGS", [Net::IMAP::SEEN] }
+  end
+
+  def seen!(imap_mail)
+    puts "MARK #{imap_mail.attr['UID']} as SEEN"
+    @imap.uid_store imap_mail.attr["UID"], "+FLAGS", [Net::IMAP::SEEN]
+  end
+end

lib/protocols/mail.rb

@@ -0,0 +1,197 @@
+require "uuid"
+
+class Protocols::Mail
+  attr_accessor :from_name, :from, :to_name, :to, :subject, :body
+  DATE_FORMAT = "%a, %d %b %Y %H:%M:%S %z".freeze
+
+  HEADERS_KEEP = %w[Message-ID Subject From To Date In-Reply-To References Content-Type Content-Transfer-Encoding User-Agent MIME-Version]
+  USER_AGENT = "Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}) Mailinglist.rb/1.0"
+
+  def initialize(imap_mail: nil)
+    if imap_mail
+      @imap_mail = imap_mail
+      envelope = imap_mail.attr[Protocols::ENVELOPE]
+      body_head = imap_mail.attr[Protocols::HEADERS]
+      body_text = imap_mail.attr[Protocols::BODYTEXT]
+      @from_name = envelope.from.first.name
+      @to_name   = envelope.to.first.name
+      @from      = "#{envelope.from.first.mailbox}@#{envelope.from.first.host}"
+      @to        = "#{envelope.to.first.mailbox}@#{envelope.to.first.host}"
+      @subject   = envelope.subject
+      @body      = body_text
+      @headers   = parse_rfc822_headers!(body_head)
+      @seq       = header("X-Sequence").to_i
+      @message_id = header("Message-id")
+    end
+  end
+
+  def clone
+    Protocols::Mail.new(imap_mail: @imap_mail)
+  end
+
+  def seen!(imap_client:)
+    if @imap_mail
+      imap_client.seen!(@imap_mail)
+    else
+      false
+    end
+  end
+
+  # Transform an received email into a mailing list email.
+  # Tested with evolution. Thunderbird bug at the moment.
+  #
+  # @param list   [Mailinglist]
+  # @param dest [Email]
+  def to_redistribute(list:, dest:)
+    new = clone
+    new.replace_headers!(
+      ["User-Agent",       USER_AGENT],
+      # require people to do not respond privatly
+      ["Reply-to",         "#{list.name} <#{list.email}>"],
+      ["Errors-To",        list.email],
+      ["To",               dest.email],
+      ["From",             "\"#{@from_name}\" (via #{list.name}) <#{@from}>"],
+      ["Sender",           @from],
+      ["Date",             Time.now.strftime(DATE_FORMAT)],
+      ["List-Id",          "<#{list.email}>"],
+      ["List-Post",        "<mailto:#{list.email}>"],
+      ["List-Help",        "<mailto:#{list.help_email}>"],
+      ["List-Subscribe",   "<mailto:#{list.subscribe_email}>"],
+      ["List-Unsubscribe", "<mailto:#{list.unsubscribe_email}>"],
+      ["List-Post",        "<mailto:#{list.post_email}>"],
+      ["List-Owner",       "<mailto:#{list.owner_email}>"],
+      ["In-Reply-To",      @message_id],
+      ["Precedence",       "list"],
+      # ["Precedence",       "bulk"],
+      ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
+      ["X-Sequence",         (@seq + 1).to_s],
+      ["X-Loop",           list.email],
+      ["Return-Path",      "<#{list.email}>"],
+    )
+
+    new.to   = dest.email
+    new.from = list.email
+    new.body = "#{body}#{list.signature}"
+
+    new
+  end
+
+  # Create a email from scratch.
+  #
+  # @param list [Mailinglist]
+  # @param to   [String]        email of the target
+  # @param cc   [Array(String)] optional, emails (["mail@host", "Name <mail@host>", ...])
+  def self.build(subject:, list:, to:, body:, cc: [])
+    new = new()
+    new.replace_headers!(
+      ["User-Agent",       USER_AGENT],
+      # require people to do not respond privatly
+      ["Reply-to",         "#{list.name} <#{list.email}>"],
+      ["Errors-To",        list.email],
+      ["To",               to],
+      ["Subject",          subject],
+      ["From",             "\"#{list.name}\" <#{list.email}>"],
+      ["Sender",           list.email],
+      ["Date",             Time.now.strftime(DATE_FORMAT)],
+      ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
+    )
+    new.replace_headers!(["CC", cc.join(", ")]) if !cc.empty?
+
+    new.subject = subject
+    new.to   = to
+    new.from = list.email
+    new.body = "#{body}#{list.signature}"
+    new
+  end
+
+  # Create a email from scratch.
+  # Create a response email for an existing one.
+  #
+  # @param list [Mailinglist]
+  # @param to   [Protocols::Mail]
+  def to_response(list:, to:, body:)
+    new = clone
+    new.replace_headers!(
+      ["User-Agent",       USER_AGENT],
+      # require people to do not respond privatly
+      ["Reply-to",         "#{list.name} <#{list.email}>"],
+      ["Errors-To",        list.email],
+      ["To",               list.email],
+      ["From",             "\"#{list.name}\" <#{list.email}>"],
+      ["Sender",           list.email],
+      ["Date",             Time.now.strftime(DATE_FORMAT)],
+      ["Message-Id",       "<#{UUID.generate}@#{ENV['SENDER_HOST']}>"],
+    )
+
+    new.to   = to.from
+    new.from = list.email
+    new.body = "#{body}#{list.signature}"
+    new
+  end
+
+  def parse_rfc822_headers!(rfc822)
+    headers = []
+    rfc822.split("\r\n").each do |line|
+      if line[0] == "\t"
+        headers.last[1] << "\r\n" << line
+      else
+        k, v = line.split(":", 2)
+        headers << [k, v.to_s.strip]
+      end
+    end
+    headers.filter! { HEADERS_KEEP.include?(_1[0]) }
+  end
+
+  def header(k)
+    tuple = headers.find { _1[0].downcase == k.downcase }
+    tuple && tuple[1]
+  end
+
+  def headers
+    @headers ||= []
+  end
+
+  # @params new_headers is a [ [key, value], ...] array
+  def add_headers!(*new_headers)
+    new_headers.each { headers << _1 }
+  end
+
+  # @param headers_keys is a [word, word, ...] array
+  def delete_headers!(*headers_keys)
+    headers_keys.each { delete_header! _1 }
+  end
+
+  def delete_header!(header_key)
+    headers.delete_if { _1[0].downcase == header_key.downcase }
+  end
+
+  def replace_headers!(*update_headers)
+    delete_headers!(*update_headers.map(&:first))
+    add_headers!(*update_headers)
+  end
+
+  private def smtp_headers
+    headers.map { _1.join(": ") }.join("\r\n")
+  end
+
+  # TODO
+  # List-Id: <mailto:>
+  # List-Help: <mailto:>
+  # List-Subscribe: <mailto:>
+  # List-Unsubscribe: <mailto:>
+  # List-Post: <mailto:>
+  # List-Owner: <mailto:>
+  # List-Archive: <https://...>
+  # Archived-At: <https://...>
+  # List-Post: NO (posting not allowed on this list)
+
+  # https://www.ietf.org/rfc/rfc2369.txt
+  # https://www.rfc-editor.org/rfc/rfc5983
+  def to_smtp
+    [
+      "#{smtp_headers}\r\n\r\n#{@body}",
+      @from,
+      @to,
+    ]
+  end
+end

lib/protocols/smtp.rb

@@ -0,0 +1,41 @@
+require "net/smtp"
+
+class Protocols::Smtp
+  def initialize
+    reset_smtp_client!
+  end
+
+  def distribute(mail)
+    puts "SEND #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    smtp_raw = mail.to_smtp
+    $logger.debug smtp_raw.join("=====")
+    send_message_safe(*smtp_raw)
+  rescue => err
+    warn "FAILED #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
+    $logger.debug err
+  end
+
+  private def reset_smtp_client!
+    @smtp = Net::SMTP.new(
+      ENV["SMTP_HOST"], ENV["SMTP_PORT"], tls: ENV["SMTP_TLS"] == "true",
+    ).start(
+      user: ENV["SMTP_USER"],
+      secret: ENV["SMTP_PASSWORD"],
+      authtype: :login,
+    )
+  end
+
+  def send_message_safe(*raw, max_tries: 3)
+    if max_tries == 0
+      return $logger.error("send_message_safe reached max_tries_limit")
+    end
+
+    begin
+      @smtp.send_message(*raw)
+    rescue EOFError, Net::SMTPServerBusy => err
+      warn err.message
+      reset_smtp_client!
+      send_message_safe(*raw, max_tries: max_tries - 1)
+    end
+  end
+end

lib/semver.rb

@@ -0,0 +1,44 @@
+class Semver
+  def initialize(string)
+    @values = string.gsub(/^v(\d.+)/, '\1').split(".").map(&:to_i)
+  end
+
+  def [](index)
+    @values[index] || 0
+  end
+
+  def to_s
+    "v#{@values.join(".")}"
+  end
+
+  def <=>(left)
+    cmp = 0
+    @values.each_with_index do |value, index|
+      cmp = value - left[index]
+      break if cmp != 0
+    end
+    cmp
+  end
+
+  def >(left)
+    (self <=> left) > 0
+  end
+
+  def <(left)
+    (self <=> left) < 0
+  end
+
+  def ==(left)
+    (self <=> left) == 0
+  end
+
+  def !=(left)
+    (self <=> left) != 0
+  end
+end
+
+# pp Utils::Semver.new("1.0") < Utils::Semver.new("1.1")
+# pp Utils::Semver.new("0.1") < Utils::Semver.new("1.1")
+# pp Utils::Semver.new("1.1") < Utils::Semver.new("1.2")
+# pp Utils::Semver.new("1.1") < Utils::Semver.new("2.0")
+# pp Utils::Semver.new("12.0") > Utils::Semver.new("2.2")

lib/web.rb

@@ -0,0 +1,2 @@
+require "sinatra"
+require_relative "web/mailinglists"

lib/web/mailinglists.rb

@@ -0,0 +1,9 @@
+get "/" do
+  mailinglists = Mailinglist.where(strategy: %w[free validated]).all
+  slim :index, locals: {mailinglists:}
+end
+
+get "/mailinglist/:id" do
+  mailinglist = Mailinglist.first(id: params[:id], strategy: %w[free validated])
+  slim :mailinglist, locals: {mailinglist:}
+end

lib/web/views/index.slim

@@ -0,0 +1,10 @@
+h1
+  | Index
+p
+  ul
+  - for mailinglist in mailinglists
+    li
+      a href="/mailinglist/#{mailinglist.id}"
+        strong
+          | #{mailinglist.name}
+      | : #{mailinglist.email}

lib/web/views/layout.slim

@@ -0,0 +1,8 @@
+doctype html
+html
+  head
+    title
+      | Mailing List
+
+  body
+    == yield

lib/web/views/mailinglist.slim

@@ -0,0 +1,9 @@
+h1
+  | Mailinglist #{mailinglist.name}
+p
+  ul
+  - for email in mailinglist.emails
+    li
+      strong
+        | #{email.name}
+      | : #{email.email}

server.rb

@@ -1,96 +0,0 @@
-require "net/smtp"
-require "net/imap"
-require "dotenv"
-require "uuid"
-require "pry"
-
-Dotenv.load!
-
-class MailingList
-  class Smtp
-    def initialize
-      @smtp = Net::SMTP.new(
-        ENV["SMTP_HOST"], ENV["SMTP_PORT"], tls: ENV["SMTP_TLS"] == "true",
-      ).start(
-        user: ENV["SMTP_USER"],
-        secret: ENV["SMTP_PASSWORD"],
-        authtype: :login,
-      )
-    end
-
-    def distribute(*mail)
-      @smtp.send_message(*mail)
-    end
-  end
-
-  class Imap
-    def initialize
-      @imap = Net::IMAP.new(ENV["IMAP_HOST"], ENV["IMAP_PORT"], ssl: true)
-      @imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
-      @imap.select("INBOX")
-    end
-
-    BODY = "RFC822".freeze
-    ENVELOPE = "ENVELOPE".freeze
-    def fetch
-      @imap.check
-      # puts "DEBUG: check"
-      id = @imap.search(["NOT", "SEEN"]).first
-      return nil if id.nil?
-      # puts "DEBUG: id=#{id}"
-      message_raw = @imap.fetch(id, [ENVELOPE, BODY]).first
-      return nil if message_raw.nil?
-      message = message_raw.attr
-      # puts "DEBUG: message found"
-      from_name = message[ENVELOPE].from.first.name
-      to_name = message[ENVELOPE].to.first.name
-      from = "#{message[ENVELOPE].from.first.mailbox}@#{message[ENVELOPE].from.first.host}"
-      to = "#{message[ENVELOPE].to.first.mailbox}@#{message[ENVELOPE].to.first.host}"
-      subject = message[ENVELOPE].subject
-      body = message[BODY].split("\r\n\r\n", 2).last
-      # puts "DEBUG: generate mail"
-      mail = Mail.new(from_name:, from:, to_name:, to:, subject:, body:)
-
-      @imap.store id, "+FLAGS", [Net::IMAP::SEEN]
-      puts "#{from}\t -> #{to}:\t#{subject}"
-      # puts "DEBUG: set as seen"
-      mail
-    end
-  end
-
-  class Mail
-    attr_accessor :from_name, :from, :to_name, :to, :subject, :body
-
-    def initialize(from_name:, from:, to_name:, to:, subject:, body:)
-      @from_name = from_name
-      @from = from
-      @to_name = to_name
-      @to = to
-      @subject = subject
-      @body = body
-    end
-
-    def to_smtp_distribution(to:)
-      [
-"From: Mailinglist <#{@to}>
-To: <#{to}>
-Subject: #{@subject}
-Date: #{Time.now.to_s}
-Message-Id: <#{UUID.generate}@example.com>
-
-#{@body}",
-        @to,
-        to,
-      ]
-    end
-  end
-end
-
-smtp = MailingList::Smtp.new
-imap = MailingList::Imap.new
-
-while true
-  mail = imap.fetch
-  smtp.distribute(*mail.to_smtp_distribution(to: "arthur.poulet.test@sceptique.eu")) if mail
-  sleep 1
-end